//--------------------------------------------------------------------------------------------------------------------------------------------------------------
public static DatabaseSearchPattern GenerateSearchPatternInteger(int min, int max, int numTestsInGroup)
{
DatabaseSearchPattern rsp = null;
//-----a----- Get the expression type (==, <=, >= or between)
int randomET = GenerateExpressionType(numTestsInGroup);
//-----b----- Generate the range
int range = TestParameters.SearchRangeSize; // 50,000
//-----c----- We want to generalise for multiple tests --- so if numTests > 0, then increase the range and bring in the min and max to ensure a larger range .....
if (numTestsInGroup > 1)
{
range = range * 10;
// For >= or <= lets also tweak the min and max to ensure the range is larger
if (randomET == 2)
{
min = min + range;
max = max - range;
}
}
//-----d----- Generate the random value
int randomVal = MGLEncryption.GenerateRandomInt(min, max);
//-----e----- Now generate the search pattern
if (randomET == 1) //_____ == _____
{
rsp = new DatabaseSearchPattern("TestInt", randomVal);
}
else if (randomET == 2) //_____ >= or <= range search _____
{
bool randomBool = MGLEncryption.GenerateRandomBool();
if (randomBool == true)
{
rsp = new DatabaseSearchPattern("TestInt", MGLEncryption.GenerateRandomInt(max - range, max), randomBool);
}
else
{
rsp = new DatabaseSearchPattern("TestInt", MGLEncryption.GenerateRandomInt(min, min + range), randomBool);
}
}
else //_____ Between _____
{
rsp = new DatabaseSearchPattern("TestInt", randomVal, randomVal + range);
}
return(rsp);
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates a series of simple integer range searches... useful for more specific testing
/// </summary>
public static void TestSorting()
{
List <DatabaseIndexInfo> riis = rw.IndexFindAll(new TestObj());
int randomVal = MGLEncryption.GenerateRandomInt(1, 1000000);
long randomVal2 = MGLEncryption.GenerateRandomLong(8000000000, 12000000000);
double randomVal3 = MGLEncryption.GenerateRandomDouble(0, 1);
DateTime randomVal4 = MGLEncryption.GenerateRandomDateTime(TestParameters.DateMin, TestParameters.DateMax);
List <DatabaseSearchPattern> testPatterns = new List <DatabaseSearchPattern>()
{
// Bool
new DatabaseSearchPattern("TestBool", MGLEncryption.GenerateRandomBool()),
// Score - equivalent (int)
new DatabaseSearchPattern("TestInt", randomVal),
// Score - greater than
new DatabaseSearchPattern("TestInt", randomVal, true),
// Score - less than
new DatabaseSearchPattern("TestInt", randomVal, false),
// Score - between
new DatabaseSearchPattern("TestInt", randomVal, randomVal + TestParameters.SearchRangeSize),
// Text - Starts with
new DatabaseSearchPattern("TestStr", MGLEncryption.GetSalt(RedisWrapper.TextIndexLength).ToString().ToLower()),
// Score - between (long)
new DatabaseSearchPattern("TestLong", randomVal2, randomVal2 + (4000000000 * TestParameters.SearchRangeSize / 1000000)),
// Primary key
new DatabaseSearchPattern((long)randomVal),
// Score - between
new DatabaseSearchPattern("TestDouble", randomVal3, randomVal3 + ((double)TestParameters.SearchRangeSize / 1000000.0)),
// Date time - betwen
new DatabaseSearchPattern("TestDT", randomVal4, randomVal4.AddDays(1)),
// Date time - less than
new DatabaseSearchPattern("TestDT", randomVal4, false),
// Date time - greater than
new DatabaseSearchPattern("TestDT", randomVal4, true),
// Date time - equivalent
new DatabaseSearchPattern("TestDT", randomVal4)
};
// sort it
testPatterns.Sort(DatabaseSearchPattern.Sort(riis));
string isItGucci = "";
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates a random number between 1 and 10; normally 1 will be used for equivalence, 2 for less than or greater than and the remainder for between.
/// A special case arises with the equivalence - we only want to use this if there is only one test (this one) in the group of search patterns
/// </summary>
public static int GenerateExpressionType(int numTestsInGroup)
{
// We will need to randomly choose the expression type =, >=, <=, BETWEEEN, with only a few results as equivalent
// For most of the range queries 1 will be equivalence (and only if there is only one search pattern), 2 is <= or >= and anything else is between...
// We want to weight heavily to the between queries as these are truely random and will pull out results from the full range, rather than just the top and tail ..
int randomET = MGLEncryption.GenerateRandomInt(1, 10);
// If there are more than one query do not go for the equivalence as these will generate too few results...
if (randomET == 1 && numTestsInGroup > 1)
{
while (randomET == 1)
{
randomET = MGLEncryption.GenerateRandomInt(1, 10);
}
}
return(randomET);
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates a number of randomly generated RedisTestObj objects, with each parameter between a range...
/// These are ONLY used for writing, so we can ignore if the readonly flag is enabled
/// </summary>
public static void GenerateRandomObjects()
{
// These are ONLY used for writing, so we can ignore if the readonly flag is enabled
if (TestParameters.DoWrite == false)
{
Logger.Log("Skipping generating the random objects as the global TestParameter.JustDoRead parameter is set to read only.");
return;
}
RandomObjects = new List <object>();
// Generate some random data!
Logger.LogSubHeading("Generating " + NumIterations + " randomly generated TestObj objects...");
int i = 0;
for (i = 0; i < NumIterations; i++)
{
RandomObjects.Add(
// The primary key
new TestObj((ulong)i,
// A random int
MGLEncryption.GenerateRandomInt(1, MaxIntegerInRange),
// longs ...
MGLEncryption.GenerateRandomLong(TestParameters.MinLongInRange, TestParameters.MaxLongInRange),
// doubles ...
MGLEncryption.GenerateRandomDouble(TestParameters.MinDoubleInRange, TestParameters.MaxDoubleInRange),
// bools ...
MGLEncryption.GenerateRandomBool(),
// strings ...
MGLEncryption.GetSalt(20).ToString(),
// DateTime - Standardise the dates to the nearest second so that the MySQL vs Redis comparison is consistent
// Otherwise Redis is stored to the nearest MS while MySQL only stores to the nearest second
DatabaseSearchPattern.StandardiseDateToStartOfSecond(MGLEncryption.GenerateRandomDateTime(TestParameters.DateMin, TestParameters.DateMax), false)));
// Log the progress ...
Logger.Log(i, 1000, NumIterations);
}
Logger.Log(i, 1, NumIterations);
Logger.Log("\n");
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Tests the combining of lists as intersection or union operations ...
/// </summary>
public static void TestListCombining()
{
Logger.LogSubHeading("Testing the List combining algorithm ...");
// ok three lists of 1000, 10000 and 100000 done multiple times
// Also try to increase this by a factor of 10 too ..
/*
* Note that the lists wont exactly be this length because of possible duplicates:
* Q: How many iterations does it take to cover a range with random values?
* A: This is known as the coupon collector's problem. See the link for details and references.
* Asymptotically, the expected number of trials needed grows as n ln n (plus lower order terms).
* ln is the natural logarithm - see https://en.wikipedia.org/wiki/Natural_logarithm
*
* More on the coupon collectors problem - https://en.wikipedia.org/wiki/Coupon_collector%27s_problem
* For example, when n = 50 it takes about 225 trials to collect all 50 coupons.
* This is calculated as 1/probability1 + 1/probability2 + 1/probabilityN e.g. 1/ (Num already chosen/Total Number)
* The upshot is that 1000 iterations of a range of randomly generated numbers between 1 and 1000, is likely to produce around 631 unique values. For 10,000, its around 6,320...
* So how close we get to this (not too high OR low), is a good indication that our random number generator is on the money....
*/
int numIterations = 100;
int list1Length = 1000000; // 100000;
int list2Length = 10000; // 10000;
int list3Length = 100000; // 1000000;
int rangeMin = 1;
int rangeMax = 1000000;
RedisWrapper rw = new RedisWrapper();
Logger.Log("Running " + numIterations + " iterations with three lists of " + list1Length.ToString("N0") + ", "
+ list2Length.ToString("N0") + " and " + list3Length.ToString("N0") + " randomly generated integers ...");
DateTime start = DateTime.Now;
for (int i = 0; i < numIterations; i++)
{
//-----a----- Generate the random data ...
Logger.Log("Iteration " + i + " - Generating the random data ...");
List <List <uint> > listOfLists = new List <List <uint> >();
List <uint> outputIntersection = new List <uint>();
List <uint> outputUnion = new List <uint>();
// List 1 ... we use hashsets to ensure no duplicates...
HashSet <uint> list1 = new HashSet <uint>();
for (int j = 0; j < list1Length; j++)
{
list1.Add((uint)MGLEncryption.GenerateRandomInt(rangeMin, rangeMax));
}
listOfLists.Add(list1.ToList());
// List 2 ...
HashSet <uint> list2 = new HashSet <uint>();
for (int j = 0; j < list2Length; j++)
{
list2.Add((uint)MGLEncryption.GenerateRandomInt(rangeMin, rangeMax));
}
listOfLists.Add(list2.ToList());
// List 3 ...
HashSet <uint> list3 = new HashSet <uint>();
for (int j = 0; j < list3Length; j++)
{
list3.Add((uint)MGLEncryption.GenerateRandomInt(rangeMin, rangeMax));
}
listOfLists.Add(list3.ToList());
Logger.Log("Iteration " + i + " has the following number of unique values for each list: " + list1.Count.ToString("N0") + ", "
+ list2.Count.ToString("N0") + " and " + list3.Count.ToString("N0") + ".");
// Now combine the lists .... Intersection
outputIntersection = DataStructures.CombineLists(listOfLists, true);
// And union
outputUnion = DataStructures.CombineLists(listOfLists, false);
Logger.Log("Iteration " + i + " had " + outputIntersection.Count + " matching by intersection and " + outputUnion.Count + " by union.");
}
TimeSpan procTime = DateTime.Now.Subtract(start);
Logger.Log("Completed testing the combining of lists in " + procTime.TotalMilliseconds.ToString("N0") + " seconds.");
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates n search patterns with between 1 and 3 search parameters randomly chosen from 6 of the RedisTestObj attributes
/// 11-Aug-2016 - updated so that the patterns we are searching for are broader if there are multiple tests ...
/// 16-Aug-2016 - No longer searching using the TestBool parameter as the query results are far too broad.
/// </summary>
public static void GenerateRandomPatterns(List <DatabaseIndexInfo> riis)
{
//-----0----- Reset the list of patterns to search for
PatternsToSearchFor = new List <List <DatabaseSearchPattern> >();
// 1=PrimaryKey, 2=Integer, 3=Long, 4=Double, 5=String, 6=DateTime (Note we are not using bool searches here as they are far too sloooow)
int totalNumPatterns = 6;
//-----1----- Process the number of search pattern groups specified in the global parameter
for (int i = 0; i < NumSearchPatterns; i++)
{
List <DatabaseSearchPattern> rsps = new List <DatabaseSearchPattern>();
//-----2----- Determine how many tests to run and from which parameters...
int numTests = MGLEncryption.GenerateRandomInt(1, 3);
List <int> testTypes = new List <int>();
for (int j = 0; j < numTests; j++)
{
int testType = MGLEncryption.GenerateRandomInt(1, totalNumPatterns);
//-----2a----- Reduce the number of PK queries in the multiple search parameter queries and instead choose integer range parameters..
if (testType == 1 && j > 0)
{
testType = 2;
}
//-----2b----- If this test type already exists, then iterate through and find a test type that has not yet already been chosen
while (testTypes.Contains(testType) == true)
{
testType = MGLEncryption.GenerateRandomInt(1, totalNumPatterns);
// Reduce the number of PK queries in the multiple search parameter queries and instead choose integer range parameters..
if (testType == 1 && j > 0)
{
testType = 2;
}
}
testTypes.Add(testType);
//-----3----- Now go through and randomly generate the SearchPattern for each of these test types
// We now have about six types of test we are processing...
if (testType == 1) //-----a----- ID - Primary Key - Equivalent - individual result ...
{
rsps.Add(GenerateSearchPatternPrimaryKey(0, TestParameters.NumIterations));
}
else if (testType == 2) //-----b----- TestInt - Try to ensure that max results returned is 20%
{
rsps.Add(GenerateSearchPatternInteger(1, TestParameters.MaxIntegerInRange, numTests));
}
else if (testType == 3) //-----c----- TestLong - a range of 4 billion and we are searching between for ranges of 10,000,000 (i.e. 0.25%)
{
rsps.Add(GenerateSearchPatternLong(TestParameters.MinLongInRange, TestParameters.MaxLongInRange, numTests));
}
else if (testType == 4) //-----d----- TestDouble - Try to ensure that max results return is 20%
{
rsps.Add(GenerateSearchPatternDouble(TestParameters.MinDoubleInRange, TestParameters.MaxDoubleInRange, numTests));
}
else if (testType == 5) //-----e----- TestStr - max results returned is 1/(40*40) when using a two char search length
{
rsps.Add(GenerateSearchPatternString());
}
else if (testType == 6) //-----f----- TestDT - searching for ranges in hours
{
rsps.Add(GenerateSearchPatternDateTime(TestParameters.DateMin, TestParameters.DateMax, numTests));
}
}
// 19-Aug-2016 - if DoDebug == true, we need to sort here so that the views in the MySQL and Redis debug lists are consistent!
// Otherwise it is difficult to compare the results of the two databases afterwards ..
if (RedisWrapper.DoDebug == true)
{
rsps.Sort(DatabaseSearchPattern.Sort(riis));
}
// add our new search patterns ...
PatternsToSearchFor.Add(rsps);
}
}
