static void Main(string[] args)
{
string MNISTFile = null;
string MNISTData = null;
string MNISTLabels = null;
var p = new OptionSet();
bool just_accuracy = false;
bool just_loss = false;
p.Add("nnet=", "MNIST neural network file name", x => MNISTFile = x);
p.Add("datafile=", "MNIST data file name", x => MNISTData = x);
p.Add("labelfile=", "MNIST label file name", x => MNISTLabels = x);
p.Add <bool>("optimization=", "Do optimization (Default: true)", (x => RobustnessOptions.DoOptimization = x));
p.Add <double>("bound=", "Linfinity-ball to search", (x => RobustnessOptions.Epsilon = x));
p.Add <double>("sub=", "Subsample from 'live' constraints (0.0-1.0)", (x => RobustnessOptions.LiveConstraintSamplingRatio = x));
p.Add <string>("registry=", "Unique name to store output examples and statistics", (x => RobustnessOptions.Registry = x));
p.Add <bool>("cegar=", "Do CEGAR (default: true)", (x => RobustnessOptions.CEGAR = x));
p.Add <string>("only-accuracy", "Only evaluate accuracy", (x => just_accuracy = (x != null)));
p.Add <string>("only-loss", "Only evaluate loss", (x => just_loss = (x != null)));
p.Add <string>("no-quant-safety", "Quantization integrality safety off", (x => RobustnessOptions.QuantizationSafety = (x == null)));
p.Add <string>("max-conf", "Use max-conf objective", (x => {
if (x != null)
{
RobustnessOptions.ObjectiveKind = LPSObjectiveKind.MaxConf;
}
}));
p.Add <double>("winner-diff=", "Winning label should be that much different than second best", (x => RobustnessOptions.LabelConfidenceDiff = x));
p.Add <string>("log-png", "Log png files", (x => RobustnessOptions.SavePNGCounterexamples = (x != null)));
bool only_misclass = false;
p.Add("only-filter-misclass", "Only keep the misclassifications", (x => only_misclass = (x != null)));
Cmd.RunOptionSet(p, args);
if (MNISTFile == null || MNISTData == null || MNISTLabels == null)
{
Console.WriteLine("Invalid arguments, use --help");
Environment.Exit(1);
}
RobustnessOptions.Dump();
Options.InitializeNNAnalysis();
NeuralNet nn = MNIST.GetNN(MNISTFile);
ImageDataset data = MNIST.ReadData(MNISTLabels, MNISTData, MNIST.ALL_IMAGES, 0);
if (just_accuracy)
{
NNAccuracy.GetAccuracy(nn, data.Dataset);
return;
}
if (just_loss)
{
NNAccuracy.GetLoss(nn, data.Dataset);
return;
}
if (only_misclass)
{
string filtered = RobustnessOptions.Registry + "-misclass";
Console.WriteLine("Orig {0} data", data.Dataset.Count());
var ds = NNAccuracy.KeepMisclass(nn, data.Dataset);
Console.WriteLine("Kept {0} data", ds.Count());
ImageDataset ret = new ImageDataset(ds,
MNIST.InputCoordinates.ChannelCount,
MNIST.InputCoordinates.RowCount,
MNIST.InputCoordinates.ColumnCount, true);
MNIST.WriteData(filtered + "-labels", filtered + "-images", ret);
return;
}
// NB: No snapshotting for MNIST since it never crashes ...
ImageDataset synth = Robustness.SynthesizeCounterexamplesAndStore(nn, data, x => { return; });
MNIST.WriteData(RobustnessOptions.Registry + "-synth-labels",
RobustnessOptions.Registry + "-synth-images", synth);
}
static void Main(string[] args)
{
string CifarNNFile = null;
string CifarDataBatch = null;
bool just_accuracy = false;
bool just_loss = false;
bool raw_directory = false;
var p = new OptionSet();
p.Add <string>("nnet=", "CIFAR neural network file name", (x => CifarNNFile = x));
p.Add <string>("dataset=", "CIFAR dataset file name", (x => CifarDataBatch = x));
p.Add <string>("rawdir", "If set then --dataset value should be a directory in raw directory format", (x => raw_directory = (x != null)));
p.Add <bool> ("optimization=", "Do optimization (Default: true)", (x => RobustnessOptions.DoOptimization = x));
p.Add <double>("sub=", "Subsample from 'live' constraints (0.0-1.0)", (x => RobustnessOptions.LiveConstraintSamplingRatio = x));
p.Add <string>("registry=", "Unique name to store output examples and statistics", (x => RobustnessOptions.Registry = x));
p.Add <bool> ("cegar=", "Do CEGAR (default: true)", (x => RobustnessOptions.CEGAR = x));
p.Add <string>("only-accuracy", "Only evaluate accuracy", (x => just_accuracy = (x != null)));
p.Add <string>("only-loss", "Only evaluate loss", (x => just_loss = (x != null)));
p.Add <double>("bound=", "Linfinity-ball to search", (x => RobustnessOptions.Epsilon = x));
p.Add <double>("minval=", "Minimum value of each entry", (x => RobustnessOptions.MinValue = x));
p.Add <double>("maxval=", "Maximum value of each entry", (x => RobustnessOptions.MaxValue = x));
p.Add <string>("no-quant-safety", "Quantization integrality safety off", (x => RobustnessOptions.QuantizationSafety = (x == null)));
p.Add <double>("scale-preprocessed=", "If image data is preprocessed, scale before dumping to registry", (x => RobustnessOptions.ScalePreProcessed = x));
p.Add <double>("offset-preprocessed=", "If image data is preprocessed, offset scaled before dumping to registry", (x => RobustnessOptions.OffsetPreProcessed = x));
p.Add <string>("max-conf", "Use max-conf objective", (x =>
{
if (x != null)
{
RobustnessOptions.ObjectiveKind = LPSObjectiveKind.MaxConf;
}
}));
p.Add <double>("winner-diff=", "Winning label should be that much different than second best", (x => RobustnessOptions.LabelConfidenceDiff = x));
p.Add <string>("log-png", "Log png files", (x => RobustnessOptions.SavePNGCounterexamples = (x != null)));
bool only_filter = false;
double filter_conf = 0.98;
p.Add("only-filter", "Only filter by confidence", (x => only_filter = (x != null)));
p.Add <double>("filter-conf=", "Filter confidence", (x => filter_conf = x));
Cmd.RunOptionSet(p, args);
if (CifarNNFile == null || CifarDataBatch == null)
{
Console.WriteLine("Invalid arguments, use --help");
Environment.Exit(1);
}
/* Initialize parameters */
Options.InitializeNNAnalysis();
NeuralNet nn = CIFAR.GetNN(CifarNNFile);
ImageDataset data;
if (raw_directory)
{
// our raw data format (see lmdb2raw.py)
data = CIFAR.ReadDirectoryData(CifarDataBatch);
}
else
{
// Plain old CIFAR binary format
data = CIFAR.ReadData(CifarDataBatch, CIFAR.ALL_IMAGES, 0);
}
if (just_accuracy)
{
NNAccuracy.GetAccuracy(nn, data.Dataset);
return;
}
if (just_loss)
{
NNAccuracy.GetLoss(nn, data.Dataset);
return;
}
if (only_filter)
{
string filtered = RobustnessOptions.Registry + "-filtered-" + filter_conf.ToString();
Console.WriteLine("Orig {0} data", data.Dataset.Count());
var ds = NNAccuracy.KeepAboveConfidenceThreshold(nn, data.Dataset, filter_conf);
Console.WriteLine("Kept {0} data", ds.Count());
ImageDataset ret = new ImageDataset(ds,
CIFAR.InputCoordinates.ChannelCount,
CIFAR.InputCoordinates.RowCount,
CIFAR.InputCoordinates.ColumnCount, true);
CIFAR.WriteData(filtered, ret);
return;
}
RobustnessOptions.Dump();
string synthImagesName = RobustnessOptions.Registry + "-synth";
int labelcount = data.Dataset.LabelCount();
ImageDataset acc = new ImageDataset(new Dataset(labelcount),
CIFAR.InputCoordinates.ChannelCount,
CIFAR.InputCoordinates.RowCount,
CIFAR.InputCoordinates.ColumnCount, true);
int state = 0;
Action <LabelWithConfidence> snapshot = x =>
{
acc.Dataset.Data.Add(new MemAccessor <double[]>(x.datum));
acc.Dataset.Labels.Add(new MemAccessor <int>(x.actualLabel));
state++;
if (state >= 4)
{
CIFAR.WriteData(synthImagesName, acc);
state = 0;
}
};
ImageDataset synth = Robustness.SynthesizeCounterexamplesAndStore(nn, data, snapshot);
if (synth.Dataset.Count() == 0)
{
Console.WriteLine("Did not synthesize any counterexamples, nothing to dump ...");
return;
}
if (raw_directory)
{
throw new NotImplementedException("Output to raw directory format not yet implemented!");
}
else
{
CIFAR.WriteData(RobustnessOptions.Registry + "-synth", synth);
}
}