/// <summary>
/// Initialize the experiment with some optional XML configutation data.
/// </summary>
public void Initialize(string name, XmlElement xmlConfig)
{
_name = name;
_populationSize = XmlUtils.GetValueAsInt(xmlConfig, "PopulationSize");
_specieCount = XmlUtils.GetValueAsInt(xmlConfig, "SpecieCount");
_activationScheme = ExperimentUtils.CreateActivationScheme(xmlConfig, "Activation");
_complexityThreshold = XmlUtils.TryGetValueAsInt(xmlConfig, "ComplexityThreshold");
_description = XmlUtils.TryGetValueAsString(xmlConfig, "Description");
_parallelOptions = ExperimentUtils.ReadParallelOptions(xmlConfig);
_guesses = XmlUtils.GetValueAsInt(xmlConfig, "Guesses");
Hashed = XmlUtils.TryGetValueAsBool(xmlConfig, "Hashed").HasValue ? XmlUtils.GetValueAsBool(xmlConfig, "Hashed") : false;
ValidationGuesses = XmlUtils.GetValueAsInt(xmlConfig, "ValidationGuesses");
// Load the passwords from file
string pwdfile = XmlUtils.TryGetValueAsString(xmlConfig, "ValidationPasswordFile");
if (pwdfile != null)
{
Console.Write("Loading passwords from [{0}]...", pwdfile);
if (_passwords == null || _passwords.Count == 0)
{
int?pwLength = XmlUtils.TryGetValueAsInt(xmlConfig, "PasswordLength");
if (pwLength.HasValue)
{
Console.Write("Filtering to {0}-character passwords...", pwLength.Value);
}
_passwords = PasswordUtil.LoadPasswords(pwdfile, pwLength);
}
else
{
Console.WriteLine("WARNING: Not loading passwords for experiment (already set)");
}
}
else
{
Console.WriteLine("WARNING: Not loading passwords for experiment (not provided in config file)");
}
_eaParams = new NeatEvolutionAlgorithmParameters();
_eaParams.SpecieCount = _specieCount;
_neatGenomeParams = new NeatGenomeParameters();
_neatGenomeParams.FeedforwardOnly = false;
_neatGenomeParams.AddNodeMutationProbability = 0.03;
_neatGenomeParams.AddConnectionMutationProbability = 0.05;
// TODO: Load states from XML config file
// Generates all the valid states in the MC using all viable ASCII characters
var stateList = new List <string>();
for (uint i = 32; i < 127; i++)
{
stateList.Add(((char)i).ToString());
}
stateList.Add(null);
_states = stateList.ToArray();
_activationFnLibrary = MarkovActivationFunctionLibrary.CreateLibraryMc(_states);
}
// Loads all the passwords and the configuration file.
static void PrepareMarkovModelRuns(string results_file)
{
const string PASSWORD_OFFSET = @"../../../passwords/";
_datasetFilenames = new PasswordDatasetInfo[]
{
new PasswordDatasetInfo()
{
Filename = "faithwriters-withcount.txt", Name = "faithwriters"
},
new PasswordDatasetInfo()
{
Filename = "myspace-filtered-withcount.txt", Name = "myspace"
},
new PasswordDatasetInfo()
{
Filename = "phpbb-withcount.txt", Name = "phpbb"
},
new PasswordDatasetInfo()
{
Filename = "rockyou-withcount.txt", Name = "rockyou"
},
new PasswordDatasetInfo()
{
Filename = "singles.org-withcount.txt", Name = "singles.org"
},
// new PasswordDatasetInfo() { Filename = "morphed_english.txt", Name = "training" },
// new PasswordDatasetInfo() { Filename = "forced_morphed_english.txt", Name = "testing" }
};
Console.WriteLine("Loading all {0} password datasets...", _datasetFilenames.Length);
_passwords = new Dictionary <string, PasswordInfo> [_datasetFilenames.Length];
for (int i = 0; i < _passwords.Length; i++)
{
Console.WriteLine(_datasetFilenames[i].Name);
_passwords[i] = PasswordUtil.LoadPasswords(PASSWORD_OFFSET + _datasetFilenames[i].Filename);
}
Console.WriteLine("Done.");
_experiment = new PasswordEvolutionExperiment();
XmlDocument xmlConfig = new XmlDocument();
xmlConfig.Load(CONFIG_FILE);
_experiment.Passwords = _passwords[0];
_experiment.Initialize("PasswordEvolution", xmlConfig.DocumentElement);
using (TextWriter writer = new StreamWriter(results_file))
writer.WriteLine("TrainingSet,TestingSet,Accounts Cracked,Passwords Cracked,% Accounts,% Passwords");
}
static void Main(string[] args)
{
/////////////////////////////////////////////////////////////////////////////////////////
// Below are some examples of possible experiments and functions you may wish to call. //
/////////////////////////////////////////////////////////////////////////////////////////
// Morph an english word dictionary into a password database where 10% of passwords have a number
// ToyProblemUtil.MorphEnglish(ENGLISH_WORDS, MORPHED_ENGLISH_WORDS);
// Morph an english word dictionary into a password database where all passwords are at least 8 characters
// and contain at least one number
// ToyProblemUtil.MorphEnglish(ENGLISH_WORDS, FORCED_MORPHED_ENGLISH_WORDS, requireDigit: true, minLength: 8);
// Train on the no-rule morphed english words db and evolve against the morphed english db with
// the digit and length creation rules enforced.
// RunExperiment(MORPHED_ENGLISH_WORDS, MORPHED_SEED_FILE, MORPHED_CONFIG_FILE, MORPHED_RESULTS_FILE, false);
//Train on the phppb dataset and evolve against the rockyou dataset
//RunExperiment(PHPBB_DATASET, PHPBB_SEED_FILE, PHPBB_CONFIG_FILE, PHPBB_RESULTS_FILE, false);
//Train on the toyDistribution dataset and evolve against the toyDistribution dataset
//RunExperiment(TOY_DISTRIBUTION_CONFIG_FILE, false);
// Print some summary statistics about the distribution of passwords in the two morphed english dictionaries.
// PasswordUtil.PrintStats(@"../../../passwords/morphed_english.txt"); // no creation rules
// PasswordUtil.PrintStats(@"../../../passwords/forced_morphed_english.txt"); // digit and length rules
// Run a really big analysis comparing the first-order Markov model to an 8-layered one.
// PrepareMarkovModelRuns();
// Parallel.For(0, _datasetFilenames.Length, i => RunAllMarkovModelPairs(i));
// Check if a database of hashed passwords contains some common passwords (check for creation rules)
// MD5HashChecker md5 = new MD5HashChecker(@"../../../passwords/stratfor_hashed.txt");
// md5.PrintCounts();
// Load the training set passwords from file
var passwords = PasswordUtil.LoadPasswords(@"/Users/Wesley/Projects/password-evolution/passwords/morphed_english.txt", 8);
// Create a Markov model from the passwords. This model will be used
// as our seed for the evolution.
int outputs = MarkovFilterCreator.GenerateFirstOrderMarkovFilter(
@"/Users/Wesley/Projects/password-evolution/models/supervised/morphed_english.xml", passwords);
Console.WriteLine("Outputs: {0}", outputs);
}
/// <summary>
/// Takes in an English dictionary and morphs it so that some words contain numbers, and it looks more password-like.
/// Note: this probably can be improved by using a Markov model to enforce the minimum length creation rule. It
/// could also be enhanced with the ability to generate more rules or more realistic passwords.
/// </summary>
/// <param name="inFile">The English dictionary file.</param>
/// <param name="outFile">The file in which to save the morphed dictionary.</param>
/// <param name="counts">Whether to assume each password is in the dictionary once or to add counts.</param>
/// <param name="minLength">The minimum length of a password in the dictionary. If a word isn't long enough, it will have number added until it is.</param>
public static void MorphEnglish(string inFile, string outFile, bool requireDigit = false, bool counts = false, int minLength = 0)
{
// Load the passwords
var english = PasswordUtil.LoadPasswords(inFile);
// Morph the dictionary either with a required digit creation rule or not, depending on the requireDigit value
var morphed = requireDigit
? morphEnglish(english, defaultDigitProbs, requiredDigitPositionProbs, minLength) // Use a digit-required creation rule
: morphEnglish(english, defaultDigitProbs, defaultDigitPositionProbs, minLength); // No creation rule
// Write the results to file.
using (TextWriter writer = new StreamWriter(outFile))
foreach (var kv in morphed)
{
if (counts)
{
writer.WriteLine("{0} {1}", kv.Value, kv.Key);
}
else
{
writer.WriteLine(kv.Key);
}
}
}
/// <summary>
/// Creates a layered Markov filter. In this model, we have a layer of nodes
/// for each character in the password. This enables us to capture more information
/// the distribution than a simple first-order model.
/// </summary>
/// <param name="filename">The file in which to save the generated model.</param>
/// <param name="corpus">The password database from which to build the model.</param>
/// <param name="layers">The maximum number of layers (password length) to have in this model.</param>
/// <returns>The number of output nodes for SharpNEAT.</returns>
public static int GenerateLayeredMarkovFilter(string filename, string corpus, int layers)
{
var passwords = PasswordUtil.LoadPasswords(corpus);
return(GenerateLayeredMarkovFilter(filename, passwords, layers));
}
/// <summary>
/// Creates an adaptive Markov filter where we have a concept of "halting" nodes.
/// In this model, each letter node contains a probability of transitioning to
/// a halting node that simply returns the string. This removes the need to
/// hard-code how long you want your string.
/// </summary>
/// <param name="filename">The file in which to save the generated model.</param>
/// <param name="corpus">The password database from which to build the model.</param>
/// <returns>The number of output nodes for SharpNEAT.</returns>
public static int GenerateAdaptiveMarkovFilter(string filename, string corpus)
{
var passwords = PasswordUtil.LoadPasswords(corpus);
return(GenerateAdaptiveMarkovFilter(filename, passwords));
}
/// <summary>
/// Trains a Markov model on a the training set of passwords, then evolves it against the target password database
/// specified in the config file. At the end of the evolution, the champion model is evaluated for a larger number
/// of guesses.
/// </summary>
/// <param name="trainingSetFile">The file containing the passwords from which to build the initial Markov model.</param>
/// <param name="seedFile">The file to which the initial Markov model will be saved.</param>
/// <param name="configFile">The file containing all the configuration parameters of the evolution.</param>
/// <param name="resultsFile">The file to which the results will be saved at each generation.</param>
/// <param name="validateSeed">If true, the seed model will first be validated against a large number of guesses.</param>
//private static void RunExperiment(string trainingSetFile, string seedFile, string configFile, string resultsFile, bool validateSeed = false)
private static void RunExperiment(string configFile, bool validateSeed = false)
{
Console.Write("Building Markov model...");
// Load the XML configuration file
XmlDocument xmlConfig = new XmlDocument();
xmlConfig.Load(configFile);
XmlElement xmlConfigElement = xmlConfig.DocumentElement;
// Set Training File
string trainingSetFile = XmlUtils.GetValueAsString(xmlConfigElement, "TrainingFile");
// Create seedFile
string seedFile = XmlUtils.GetValueAsString(xmlConfigElement, "SeedFile");
// Create results file.
string resultsFile = XmlUtils.GetValueAsString(xmlConfigElement, "ResultsFile");
Console.WriteLine("\nTraining File: {0}\nSeed File: {1}\nResults File: {2}", trainingSetFile, seedFile, resultsFile);
// Load the training set passwords from file
var passwords = PasswordUtil.LoadPasswords(trainingSetFile, 8);
// Create a Markov model from the passwords. This model will be used
// as our seed for the evolution.
int outputs = MarkovFilterCreator.GenerateFirstOrderMarkovFilter(seedFile, passwords);
// Free up the memory used by the passwords
passwords = null;
Console.WriteLine("Done! Outputs: {0}", outputs);
_experiment = new PasswordEvolutionExperiment();
_experiment.OutputCount = outputs;
// Initialize the experiment with the specifications in the config file.
_experiment.Initialize("PasswordEvolution", xmlConfig.DocumentElement);
// Set the passwords to be used by the fitness evaluator.
// These are the passwords our models will try to guess.
// PasswordsWithAccounts is the file used for validation. Its account values won't be changed.
PasswordCrackingEvaluator.Passwords = _experiment.Passwords;
PasswordCrackingEvaluator.PasswordsWithAccounts = new Dictionary <string, double>(_experiment.Passwords); // Makes a deep copy
Console.WriteLine("Loading seed...");
// Load the seed model that we created at the start of this function
var seed = _experiment.LoadPopulation(XmlReader.Create(seedFile))[0];
// Validates the seed model by running it for a large number of guesses
if (validateSeed)
{
Console.WriteLine("Validating seed model...");
var seedModel = _experiment.CreateGenomeDecoder().Decode(seed);
ValidateModel(seedModel, _experiment.Passwords, VALIDATION_GUESSES, _experiment.Hashed);
}
// Create evolution algorithm using the seed model to initialize the population
Console.WriteLine("Creating population...");
_ea = _experiment.CreateEvolutionAlgorithm(seed);
// Attach an update event handler. This will be called at the end of every generation
// to log the progress of the evolution (see function logEvolutionProgress below).
_ea.UpdateEvent += new EventHandler(logEvolutionProgress);
//_ea.UpdateScheme = new UpdateScheme(1);//.UpdateMode.
// Setup results file
using (TextWriter writer = new StreamWriter(resultsFile))
writer.WriteLine("Generation,Champion Accounts,Champion Uniques,Average Accounts,Average Uniques,Total Accounts,Total Uniques");
_generationalResultsFile = resultsFile;
// Start algorithm (it will run on a background thread).
Console.WriteLine("Starting evolution. Pop size: {0} Guesses: {1}", _experiment.DefaultPopulationSize, _experiment.GuessesPerIndividual);
_ea.StartContinue();
// Wait until the evolution is finished.
while (_ea.RunState == RunState.Running)
{
Thread.Sleep(1000);
}
// Validate the resulting model.
var decoder = _experiment.CreateGenomeDecoder();
var champ = decoder.Decode(_ea.CurrentChampGenome);
ValidateModel(champ, _experiment.Passwords, VALIDATION_GUESSES, _experiment.Hashed);
}
/// <summary>
/// Trains a Markov model on a the training set of passwords, then evolves it against the target password database
/// specified in the config file. At the end of the evolution, the champion model is evaluated for a larger number
/// of guesses.
/// </summary>
/// <param name="trainingSetFile">The file containing the passwords from which to build the initial Markov model.</param>
/// <param name="seedFile">The file to which the initial Markov model will be saved.</param>
/// <param name="configFile">The file containing all the configuration parameters of the evolution.</param>
/// <param name="resultsFile">The file to which the results will be saved at each generation.</param>
/// <param name="validateSeed">If true, the seed model will first be validated against a large number of guesses.</param>
//private static void RunExperiment(string trainingSetFile, string seedFile, string configFile, string resultsFile, bool validateSeed = false)
private static void RunExperiment(string configFile, bool validateSeed = false)
{
Console.WriteLine("Removing previous champions...");
string[] oldChampionFiles = Directory.GetFiles(@"../../../experiments/champions/", "*.xml");
foreach (string oldChampion in oldChampionFiles)
{
File.Delete(oldChampion);
}
Console.Write("Building Markov model...");
// Load the XML configuration file
XmlDocument xmlConfig = new XmlDocument();
xmlConfig.Load(configFile);
XmlElement xmlConfigElement = xmlConfig.DocumentElement;
// Set Training File
string trainingSetFile = XmlUtils.GetValueAsString(xmlConfigElement, "TrainingFile");
// Create seedFile
string seedFile = XmlUtils.GetValueAsString(xmlConfigElement, "SeedFile");
// Create results file.
string resultsFile = XmlUtils.GetValueAsString(xmlConfigElement, "ResultsFile");
Console.WriteLine();
Console.WriteLine("Training File: {0}", trainingSetFile);
Console.WriteLine("Seed File: {0}", seedFile);
Console.WriteLine("Results File: {0}", resultsFile);
// Load the training set passwords from file
var passwords = PasswordUtil.LoadPasswords(trainingSetFile, 8);
// Create a Markov model from the passwords. This model will be used
// as our seed for the evolution.
int outputs = MarkovFilterCreator.GenerateFirstOrderMarkovFilter(seedFile, passwords);
// Free up the memory used by the passwords
passwords = null;
Console.WriteLine("Done! Outputs: {0}", outputs);
_experiment = new PasswordEvolutionExperiment();
_experiment.OutputCount = outputs;
// Initialize the experiment with the specifications in the config file.
_experiment.Initialize("PasswordEvolution", xmlConfig.DocumentElement);
// Set the passwords to be used by the fitness evaluator.
// These are the passwords our models will try to guess.
// PasswordsWithAccounts is the file used for validation. Its account values won't be changed.
PasswordCrackingEvaluator.Passwords = _experiment.Passwords;
Console.WriteLine("Loading seed...");
// Load the seed model that we created at the start of this function
var seed = _experiment.LoadPopulation(XmlReader.Create(seedFile))[0];
// Validates the seed model by running it for a large number of guesses
if (validateSeed)
{
Console.WriteLine("Validating seed model...");
var seedModel = _experiment.CreateGenomeDecoder().Decode(seed);
ValidateModel(seedModel, _experiment.Passwords, VALIDATION_GUESSES, _experiment.Hashed);
}
// Create evolution algorithm using the seed model to initialize the population
Console.WriteLine("Creating population...");
_ea = _experiment.CreateEvolutionAlgorithm(seed);
// Attach an update event handler. This will be called at the end of every generation
// to log the progress of the evolution (see function logEvolutionProgress below).
_ea.UpdateEvent += new EventHandler(logEvolutionProgress);
//_ea.UpdateScheme = new UpdateScheme(1);//.UpdateMode.
// Setup results file
using (TextWriter writer = new StreamWriter(resultsFile))
writer.WriteLine("Generation,Champion Accounts,Champion Uniques,Average Accounts,Average Uniques,Total Accounts,Total Uniques");
_generationalResultsFile = resultsFile;
// Start algorithm (it will run on a background thread).
Console.WriteLine("Starting evolution. Pop size: {0} Guesses: {1}", _experiment.DefaultPopulationSize, _experiment.GuessesPerIndividual);
_ea.StartContinue();
// Wait until the evolution is finished.
while (_ea.RunState == RunState.Running)
{
Thread.Sleep(1000);
}
if (VALIDATE_ALL_STAR)
{
// Validate the champions of each generation.
List <MarkovChain> championModels = new List <MarkovChain>();
string[] championFiles = Directory.GetFiles(@"../../../experiments/champions/", "*.xml");
foreach (string championFile in championFiles)
{
var currentChamp = _experiment.LoadPopulation(XmlReader.Create(championFile))[0];
var champModel = _experiment.CreateGenomeDecoder().Decode(currentChamp);
championModels.Add(champModel);
}
ValidateForest(championModels, _experiment.Passwords, VALIDATION_GUESSES / championFiles.Length, _experiment.Hashed);
// Validate a population made up of copies of the final champion.
/* List<MarkovChain> championCopyPop = new List<MarkovChain>();
*
* Console.WriteLine();
* Console.WriteLine("Validating the final champion population");
* for (int i = 0; i < MAX_GENERATIONS; i++)
* {
* var decoder = _experiment.CreateGenomeDecoder();
* var champ = decoder.Decode(_ea.CurrentChampGenome);
* championCopyPop.Add(champ);
* }
* ValidateAllstarTeam(championCopyPop, _experiment.Passwords, VALIDATION_GUESSES, _experiment.Hashed);
*/
}
else
{
// Validate the resulting model.
var decoder = _experiment.CreateGenomeDecoder();
var champ = decoder.Decode(_ea.CurrentChampGenome);
ValidateModel(champ, _experiment.Passwords, VALIDATION_GUESSES, _experiment.Hashed);
}
}