//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates a series of simple one pattern range searches which is useful for more specific testing
/// 1=PrimaryKey, 2=Integer, 3=Long, 4=Double, 5=String, 6=DateTime (7=Bool -- Warning this one is sloooow)
/// </summary>
public static void GenerateRandomSingleSearchPatterns(int parameterToSearchWith)
{
// Lets generate some random integers ...
PatternsToSearchFor = new List <List <DatabaseSearchPattern> >();
for (int i = 0; i < TestParameters.NumSearchPatterns; i++)
{
DatabaseSearchPattern rsp = null;
if (parameterToSearchWith == 1) // Primary key search
{
rsp = GenerateSearchPatternPrimaryKey(0, NumIterations);
}
else if (parameterToSearchWith == 2) // Integer search
{
rsp = GenerateSearchPatternInteger(1, TestParameters.MaxIntegerInRange, 1);
}
else if (parameterToSearchWith == 3) // Long search
{
rsp = GenerateSearchPatternLong(TestParameters.MinLongInRange, TestParameters.MaxLongInRange, 1);
}
else if (parameterToSearchWith == 4) // Double search
{
rsp = GenerateSearchPatternDouble(TestParameters.MinDoubleInRange, TestParameters.MaxDoubleInRange, 1);
}
else if (parameterToSearchWith == 5) // String search
{
rsp = GenerateSearchPatternString();
}
else if (parameterToSearchWith == 6) // DateTime search
{
rsp = GenerateSearchPatternDateTime(TestParameters.DateMin, TestParameters.DateMax, 1);
}
else if (parameterToSearchWith == 7) // Bool search
{
Logger.LogWarning("TestParameters.GenerateRandomIntegerRangePatterns - Boolean searches are SLOOOW and deprecated - be careful.");
TestParameters.PatternsToSearchFor.Add(new List <DatabaseSearchPattern>()
{
new DatabaseSearchPattern("TestBool", MGLEncryption.GenerateRandomBool())
});
}
else
{
Logger.LogWarning("TestParameters.GenerateRandomIntegerRangePatterns - Unknown search parameter requested - " + parameterToSearchWith);
}
// And finally - lets add our search pattern ...
TestParameters.PatternsToSearchFor.Add(new List <DatabaseSearchPattern>()
{
rsp
});
}
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
public static DatabaseSearchPattern GenerateSearchPatternLong(long min, long max, int numTestsInGroup)
{
DatabaseSearchPattern rsp = null;
//-----a----- Get the expression type (==, <=, >= or between)
int randomET = GenerateExpressionType(numTestsInGroup);
//-----b----- The range is 20m by default
long range = ((max - min) * TestParameters.SearchRangeSize / TestParameters.MaxIntegerInRange);
//-----c----- So we want to generalise for multiple tests --- so if numTests > 0, then increase the range .....
if (numTestsInGroup > 1)
{
// increase the range by a factor of 10 ...
range = range * 10;
// For >= or <= lets also tweak the min and max to ensure the range is larger
if (randomET == 2)
{
min = min + range;
max = max - range;
}
}
//-----d----- Generate the random value
long randomVal = MGLEncryption.GenerateRandomLong(min, max);
//-----e----- Generate the search expression
if (randomET == 1) //_____ == _____
{
rsp = new DatabaseSearchPattern("TestLong", randomVal);
}
else if (randomET == 2) //_____ >= or <= range search _____
// Work out if this is a <= or >= query
{
bool randomBool = MGLEncryption.GenerateRandomBool();
if (randomBool == true)
{
rsp = new DatabaseSearchPattern("TestLong", MGLEncryption.GenerateRandomLong(max - range, max), randomBool);
}
else
{
rsp = new DatabaseSearchPattern("TestLong", MGLEncryption.GenerateRandomLong(min, min + range), randomBool);
}
}
else //_____ Between _____
{
rsp = new DatabaseSearchPattern("TestLong", randomVal, randomVal + range);
}
return(rsp);
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
public static DatabaseSearchPattern GenerateSearchPatternDouble(double min, double max, int numTestsInGroup)
{
DatabaseSearchPattern rsp = null;
//-----a----- Get the expression type (==, <=, >= or between)
int randomET = GenerateExpressionType(numTestsInGroup);
//-----b----- Generate the default range size
double range = (double)TestParameters.SearchRangeSize / (double)TestParameters.MaxIntegerInRange;
//----c----- so we want to generalise for multiple tests --- so if numTests > 0, then increase the range and bring in the min and max to ensure a larger range .....
if (numTestsInGroup > 1)
{
range = range * 4;
// For >= or <= lets also tweak the min and max to ensure the range is larger
if (randomET == 2)
{
min = min + range;
max = max - range;
}
}
//-----d----- Generate the random value
double randomVal = MGLEncryption.GenerateRandomDouble(min, max);
//-----e----- Generate the search pattern
if (randomET == 1) //_____ == _____
{
rsp = new DatabaseSearchPattern("TestDouble", randomVal);
}
else if (randomET == 2) //_____ >= or <= range search _____
// Work out if this is a <= or >= query
{
bool randomBool = MGLEncryption.GenerateRandomBool();
if (randomBool == true)
{
rsp = new DatabaseSearchPattern("TestDouble", MGLEncryption.GenerateRandomDouble(max - range, max), randomBool);
}
else
{
rsp = new DatabaseSearchPattern("TestDouble", MGLEncryption.GenerateRandomDouble(min, min + range), randomBool);
}
}
else //_____ Between _____
{
rsp = new DatabaseSearchPattern("TestDouble", randomVal, randomVal + range);
}
return(rsp);
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
public static DatabaseSearchPattern GenerateSearchPatternInteger(int min, int max, int numTestsInGroup)
{
DatabaseSearchPattern rsp = null;
//-----a----- Get the expression type (==, <=, >= or between)
int randomET = GenerateExpressionType(numTestsInGroup);
//-----b----- Generate the range
int range = TestParameters.SearchRangeSize; // 50,000
//-----c----- We want to generalise for multiple tests --- so if numTests > 0, then increase the range and bring in the min and max to ensure a larger range .....
if (numTestsInGroup > 1)
{
range = range * 10;
// For >= or <= lets also tweak the min and max to ensure the range is larger
if (randomET == 2)
{
min = min + range;
max = max - range;
}
}
//-----d----- Generate the random value
int randomVal = MGLEncryption.GenerateRandomInt(min, max);
//-----e----- Now generate the search pattern
if (randomET == 1) //_____ == _____
{
rsp = new DatabaseSearchPattern("TestInt", randomVal);
}
else if (randomET == 2) //_____ >= or <= range search _____
{
bool randomBool = MGLEncryption.GenerateRandomBool();
if (randomBool == true)
{
rsp = new DatabaseSearchPattern("TestInt", MGLEncryption.GenerateRandomInt(max - range, max), randomBool);
}
else
{
rsp = new DatabaseSearchPattern("TestInt", MGLEncryption.GenerateRandomInt(min, min + range), randomBool);
}
}
else //_____ Between _____
{
rsp = new DatabaseSearchPattern("TestInt", randomVal, randomVal + range);
}
return(rsp);
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates a series of simple integer range searches... useful for more specific testing
/// </summary>
public static void TestSorting()
{
List <DatabaseIndexInfo> riis = rw.IndexFindAll(new TestObj());
int randomVal = MGLEncryption.GenerateRandomInt(1, 1000000);
long randomVal2 = MGLEncryption.GenerateRandomLong(8000000000, 12000000000);
double randomVal3 = MGLEncryption.GenerateRandomDouble(0, 1);
DateTime randomVal4 = MGLEncryption.GenerateRandomDateTime(TestParameters.DateMin, TestParameters.DateMax);
List <DatabaseSearchPattern> testPatterns = new List <DatabaseSearchPattern>()
{
// Bool
new DatabaseSearchPattern("TestBool", MGLEncryption.GenerateRandomBool()),
// Score - equivalent (int)
new DatabaseSearchPattern("TestInt", randomVal),
// Score - greater than
new DatabaseSearchPattern("TestInt", randomVal, true),
// Score - less than
new DatabaseSearchPattern("TestInt", randomVal, false),
// Score - between
new DatabaseSearchPattern("TestInt", randomVal, randomVal + TestParameters.SearchRangeSize),
// Text - Starts with
new DatabaseSearchPattern("TestStr", MGLEncryption.GetSalt(RedisWrapper.TextIndexLength).ToString().ToLower()),
// Score - between (long)
new DatabaseSearchPattern("TestLong", randomVal2, randomVal2 + (4000000000 * TestParameters.SearchRangeSize / 1000000)),
// Primary key
new DatabaseSearchPattern((long)randomVal),
// Score - between
new DatabaseSearchPattern("TestDouble", randomVal3, randomVal3 + ((double)TestParameters.SearchRangeSize / 1000000.0)),
// Date time - betwen
new DatabaseSearchPattern("TestDT", randomVal4, randomVal4.AddDays(1)),
// Date time - less than
new DatabaseSearchPattern("TestDT", randomVal4, false),
// Date time - greater than
new DatabaseSearchPattern("TestDT", randomVal4, true),
// Date time - equivalent
new DatabaseSearchPattern("TestDT", randomVal4)
};
// sort it
testPatterns.Sort(DatabaseSearchPattern.Sort(riis));
string isItGucci = "";
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates a number of randomly generated RedisTestObj objects, with each parameter between a range...
/// These are ONLY used for writing, so we can ignore if the readonly flag is enabled
/// </summary>
public static void GenerateRandomObjects()
{
// These are ONLY used for writing, so we can ignore if the readonly flag is enabled
if (TestParameters.DoWrite == false)
{
Logger.Log("Skipping generating the random objects as the global TestParameter.JustDoRead parameter is set to read only.");
return;
}
RandomObjects = new List <object>();
// Generate some random data!
Logger.LogSubHeading("Generating " + NumIterations + " randomly generated TestObj objects...");
int i = 0;
for (i = 0; i < NumIterations; i++)
{
RandomObjects.Add(
// The primary key
new TestObj((ulong)i,
// A random int
MGLEncryption.GenerateRandomInt(1, MaxIntegerInRange),
// longs ...
MGLEncryption.GenerateRandomLong(TestParameters.MinLongInRange, TestParameters.MaxLongInRange),
// doubles ...
MGLEncryption.GenerateRandomDouble(TestParameters.MinDoubleInRange, TestParameters.MaxDoubleInRange),
// bools ...
MGLEncryption.GenerateRandomBool(),
// strings ...
MGLEncryption.GetSalt(20).ToString(),
// DateTime - Standardise the dates to the nearest second so that the MySQL vs Redis comparison is consistent
// Otherwise Redis is stored to the nearest MS while MySQL only stores to the nearest second
DatabaseSearchPattern.StandardiseDateToStartOfSecond(MGLEncryption.GenerateRandomDateTime(TestParameters.DateMin, TestParameters.DateMax), false)));
// Log the progress ...
Logger.Log(i, 1000, NumIterations);
}
Logger.Log(i, 1, NumIterations);
Logger.Log("\n");
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
public static DatabaseSearchPattern GenerateSearchPatternDateTime(DateTime min, DateTime max, int numTestsInGroup)
{
DatabaseSearchPattern rsp = null;
//-----a----- Get the expression type (==, <=, >= or between)
int randomET = GenerateExpressionType(numTestsInGroup);
//-----b----- The default range in HOURS - note that we calculate this in the same manner as the other ranges from the integer parameter (approx 3.6 hrs)
int range = (int)Math.Round(TestParameters.DateMax.Subtract(TestParameters.DateMin).TotalHours *TestParameters.SearchRangeSize / TestParameters.MaxIntegerInRange);
//----c----- so we want to generalise for multiple tests --- so if numTests > 0, then increase the range and bring in the min and max to ensure a larger range .....
if (numTestsInGroup > 1)
{
range = range * 18; // becomes roughly three days // 8;
// For >= or <= lets also tweak the min and max to ensure the range is larger
if (randomET == 2)
{
min = min.AddHours(range);
max = max.AddHours(-range);
}
}
//-----d----- Generate the random value
DateTime randomVal = DatabaseSearchPattern.StandardiseDateToStartOfSecond(MGLEncryption.GenerateRandomDateTime(min, max), false);
//-----e----- And generate the search pattern as well...
if (randomET == 1) //_____ == _____
// Note that for dates equal to - would result in all the results for a specific date and time being returned - which is only very rarely going to return results
{
rsp = new DatabaseSearchPattern("TestDT", randomVal);
}
else if (randomET == 2) //_____ >= or <= range search _____
// We need a random bool to determine if this is a >= or a <= search ...
{
bool randomBool = MGLEncryption.GenerateRandomBool();
// Generate ranges between the temporarily defined max and min above and a range's width above or below these parameters
// depending on whether this is greater than or less than ...
if (randomBool == true)
{
randomVal = DatabaseSearchPattern.StandardiseDateToStartOfSecond(MGLEncryption.GenerateRandomDateTime(max.AddHours(-range), max), false);
}
else
{
randomVal = DatabaseSearchPattern.StandardiseDateToStartOfSecond(MGLEncryption.GenerateRandomDateTime(min, min.AddHours(range)), false);
}
// Now add this randomVal as a range query
rsp = new DatabaseSearchPattern("TestDT", randomVal, randomBool);
}
else //_____ Between _____
// Between our randomly generated date and a range's width above this ...
{
rsp = new DatabaseSearchPattern("TestDT", randomVal, randomVal.AddHours(range));
}
return(rsp);
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Generates n search patterns with between 1 and 3 search parameters randomly chosen from 6 of the RedisTestObj attributes
/// 11-Aug-2016 - updated so that the patterns we are searching for are broader if there are multiple tests ...
/// 16-Aug-2016 - No longer searching using the TestBool parameter as the query results are far too broad.
/// </summary>
public static void GenerateRandomPatterns(List <DatabaseIndexInfo> riis)
{
//-----0----- Reset the list of patterns to search for
PatternsToSearchFor = new List <List <DatabaseSearchPattern> >();
// 1=PrimaryKey, 2=Integer, 3=Long, 4=Double, 5=String, 6=DateTime (Note we are not using bool searches here as they are far too sloooow)
int totalNumPatterns = 6;
//-----1----- Process the number of search pattern groups specified in the global parameter
for (int i = 0; i < NumSearchPatterns; i++)
{
List <DatabaseSearchPattern> rsps = new List <DatabaseSearchPattern>();
//-----2----- Determine how many tests to run and from which parameters...
int numTests = MGLEncryption.GenerateRandomInt(1, 3);
List <int> testTypes = new List <int>();
for (int j = 0; j < numTests; j++)
{
int testType = MGLEncryption.GenerateRandomInt(1, totalNumPatterns);
//-----2a----- Reduce the number of PK queries in the multiple search parameter queries and instead choose integer range parameters..
if (testType == 1 && j > 0)
{
testType = 2;
}
//-----2b----- If this test type already exists, then iterate through and find a test type that has not yet already been chosen
while (testTypes.Contains(testType) == true)
{
testType = MGLEncryption.GenerateRandomInt(1, totalNumPatterns);
// Reduce the number of PK queries in the multiple search parameter queries and instead choose integer range parameters..
if (testType == 1 && j > 0)
{
testType = 2;
}
}
testTypes.Add(testType);
//-----3----- Now go through and randomly generate the SearchPattern for each of these test types
// We now have about six types of test we are processing...
if (testType == 1) //-----a----- ID - Primary Key - Equivalent - individual result ...
{
rsps.Add(GenerateSearchPatternPrimaryKey(0, TestParameters.NumIterations));
}
else if (testType == 2) //-----b----- TestInt - Try to ensure that max results returned is 20%
{
rsps.Add(GenerateSearchPatternInteger(1, TestParameters.MaxIntegerInRange, numTests));
}
else if (testType == 3) //-----c----- TestLong - a range of 4 billion and we are searching between for ranges of 10,000,000 (i.e. 0.25%)
{
rsps.Add(GenerateSearchPatternLong(TestParameters.MinLongInRange, TestParameters.MaxLongInRange, numTests));
}
else if (testType == 4) //-----d----- TestDouble - Try to ensure that max results return is 20%
{
rsps.Add(GenerateSearchPatternDouble(TestParameters.MinDoubleInRange, TestParameters.MaxDoubleInRange, numTests));
}
else if (testType == 5) //-----e----- TestStr - max results returned is 1/(40*40) when using a two char search length
{
rsps.Add(GenerateSearchPatternString());
}
else if (testType == 6) //-----f----- TestDT - searching for ranges in hours
{
rsps.Add(GenerateSearchPatternDateTime(TestParameters.DateMin, TestParameters.DateMax, numTests));
}
}
// 19-Aug-2016 - if DoDebug == true, we need to sort here so that the views in the MySQL and Redis debug lists are consistent!
// Otherwise it is difficult to compare the results of the two databases afterwards ..
if (RedisWrapper.DoDebug == true)
{
rsps.Sort(DatabaseSearchPattern.Sort(riis));
}
// add our new search patterns ...
PatternsToSearchFor.Add(rsps);
}
}
//--------------------------------------------------------------------------------------------------------------------------------------------------------------
/// <summary>
/// Random tests - returns a set of IDs
/// </summary>
public static List <uint> RedisPerformanceComparisonRaw()
{
bool success = false;
// Pause to enable GC
GC.Collect();
Thread.Sleep(3000);
Logger.LogSubHeading("Starting Redis performance comparison - RAW StackExchange.Redis queries ...");
HashSet <uint> objIDs = new HashSet <uint>();
int i = 0;
string objectName = TestObj.ObjectName; // "RedisTestObj";
//-----1----- Test Raw Redis
{
Logger.Log("Testing raw Redis queries ...");
Task <bool> keyExistsT = rw.DB.KeyExistsAsync(objectName + ":1");
bool keyExists = keyExistsT.Result;
Logger.Log("Firing off the queries ...");
DateTime RVTest = DateTime.Now;
List <Task <RedisValue[]> > rvTasks = new List <Task <RedisValue[]> >();
for (i = 0; i < TestParameters.NumSearchPatterns; i++)
{
// cycle through the connections and build the list of async tasks ...
DatabaseSearchPattern rsp = TestParameters.PatternsToSearchFor[i][0];
if (rsp.SearchType == SearchType.PrimaryKey)
{
// PK Query
//if (rw.DB.KeyExists("RedisTestObj:PrimaryKey", rsp.Score)) {
if (rw.DB.KeyExists(rw.KeyName(objectName, rw.ConvertID(rsp.Score))))
{
objIDs.Add(rw.ConvertID(rsp.Score));
}
}
else
{
// Range Query
//rvTasks.Add(rw.DB.SortedSetRangeByScoreAsync("{"+ objectName + ":Index:TestInt}", rsp.ScoreMin, rsp.ScoreMax));
rvTasks.Add(rw.DB.SortedSetRangeByScoreAsync("{" + objectName + ":i:" + rsp.Parameter + "}", rsp.ScoreMin, rsp.ScoreMax));
}
Logger.Log(i, 100, TestParameters.NumSearchPatterns);
}
Logger.Log(i, 1, TestParameters.NumSearchPatterns);
// Cycle through the tasks and skip over them (do not block) if they have not yet completed...
Logger.Log("");
Logger.Log("Now awaiting the results");
if (objIDs.Count == 0)
{
objIDs.UnionWith(rw.AwaitResults(rvTasks));
}
TimeSpan RVTested = DateTime.Now.Subtract(RVTest);
Logger.Log("");
Logger.Log("Total num objects = " + objIDs.Count.ToString("N0") + ".");
Logger.Log("Time to process:" + RVTested.TotalSeconds);
}
// Pause to enable GC
Logger.Log("Pausing to enable garbage collection ...");
GC.Collect();
Thread.Sleep(3000);
return(objIDs.ToList());
}
