private void testRandom(Log log, CryptoRandom.METHOD method, int nBuckets)
{
PreTest.Init();
BucketCollection col = new BucketCollection(0, 1, nBuckets);
CryptoRandom rand = new CryptoRandom(method, Guid.NewGuid().GetHashCode());
int nTotal = 1000000;
log.WriteLine("Testing (" + nBuckets.ToString() + ") " + method.ToString());
for (int i = 0; i < nTotal; i++)
{
double df = rand.NextDouble();
col.Add(df);
}
string str = "";
List <double> rgdf = new List <double>();
for (int i = 0; i < nBuckets; i++)
{
double dfPct = col[i].Count / (double)nTotal;
str += dfPct.ToString("P");
str += ", ";
rgdf.Add(dfPct);
}
str = str.TrimEnd(',', ' ');
log.WriteLine(method.ToString() + " =>> " + str);
double dfStdev = stdDev(rgdf, false);
log.WriteLine(method.ToString() + " stdev = " + dfStdev.ToString());
}
public void TestDataItem()
{
PreTest.Init();
db.stream.DataItem di = new db.stream.DataItem(10);
Log log = new Log("Test streaming database");
log.EnableTrace = true;
for (int i = 0; i < 5; i++)
{
bool bComplete = di.Add(i, i);
log.CHECK(!bComplete, "Data item should not be complete.");
}
for (int i = 5; i < 9; i++)
{
bool bComplete = di.Add(i, i);
log.CHECK(!bComplete, "Data item should not be complete.");
}
bool bComplete1 = di.Add(9, 9);
log.CHECK(bComplete1, "The data should be complete.");
double[] rgdf = di.GetData();
log.CHECK_EQ(rgdf.Length, 10, "The data length is incorrect.");
for (int i = 0; i < 10; i++)
{
log.CHECK_EQ(i, rgdf[i], "The data item at index #" + i.ToString() + " is incorrect.");
}
}
public void TestInitializationSimple()
{
PreTest.Init();
Log log = new Log("Test streaming database");
log.EnableTrace = true;
IXStreamDatabase db = new MyCaffeStreamDatabase(log);
string strSchema = "ConnectionCount=1;";
string strParam = "Connection=TestCon;Table=TestTbl;Field=TestField;";
strParam = Utility.Replace(strParam, ';', '|');
strParam = Utility.Replace(strParam, '=', '~');
strSchema += "Connection0_CustomQueryName=Test1;";
strSchema += "Connection0_CustomQueryParam=" + strParam + ";";
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery1());
string strSettings = "QueryCount=5;Start=" + DateTime.Today.ToShortDateString() + ";TimeSpanInMs=60000;SegmentSize=1;MaxCount=10;";
db.Initialize(QUERY_TYPE.SYNCHRONIZED, strSchema + strSettings);
db.Shutdown();
}
public void TestQueryGeneralText()
{
PreTest.Init();
Log log = new Log("Test streaming database with general Text data");
log.EnableTrace = true;
IXStreamDatabase db = new MyCaffeStreamDatabase(log);
try
{
string strSchema = "ConnectionCount=1;";
string strDataPath = getTestPath("\\MyCaffe\\test_data\\data\\char-rnn", true);
string strParam = "FilePath=" + strDataPath + ";";
strParam = ParamPacker.Pack(strParam);
strSchema += "Connection0_CustomQueryName=StdTextFileQuery;";
strSchema += "Connection0_CustomQueryParam=" + strParam + ";";
DateTime dt = DateTime.Today;
string strSettings = "";
db.Initialize(QUERY_TYPE.GENERAL, strSchema + strSettings);
int[] rgSize = db.QuerySize();
log.CHECK(rgSize != null, "The Query size should not be null.");
log.CHECK_EQ(rgSize.Length, 3, "The query size should have 3 items.");
log.CHECK_EQ(rgSize[0], 1, "The query size item 1 should be 1 for the number of files.");
log.CHECK_EQ(rgSize[1], 1, "The query size item 0 should be 1.");
//log.CHECK_EQ(rgSize[2], 4572882, "The query size item 2 should be 4572882 for the maximum number of characters in each of the the files.");
log.CHECK_EQ(rgSize[2], 4740059, "The query size item 2 should be 4740059 for the maximum number of characters in each of the the files.");
int nH = rgSize[1];
int nW = rgSize[2];
int nCount = nH * nW;
Stopwatch sw = new Stopwatch();
sw.Start();
SimpleDatum sd = db.Query(int.MaxValue);
SimpleDatum sdEnd = db.Query(int.MaxValue);
sw.Stop();
double dfMs = sw.Elapsed.TotalMilliseconds;
log.WriteLine("Total Time = " + dfMs.ToString() + " ms.");
log.CHECK(sdEnd == null, "The last query should be null to show no more data exists.");
//log.CHECK_EQ(sd.ItemCount, 4572882, "There should be more than one item in the data.");
log.CHECK_EQ(sd.ItemCount, 4740059, "There should be more than one item in the data.");
log.CHECK(!sd.IsRealData, "The data should be byte data, not real.");
}
finally
{
db.Shutdown();
}
}
public void TestColorMapping()
{
PreTest.Init();
Log log = new Log("Test ColorMapper");
log.EnableTrace = true;
int nCount = 300;
ColorMapper clrMap = new ColorMapper(0, 1, Color.Black, Color.Red);
double dfVal = 0;
double dfInc = 1.0 / (double)nCount;
for (int i = 0; i < nCount; i++)
{
Color clr = clrMap.GetColor(dfVal);
double dfVal1 = clrMap.GetValue(clr);
log.EXPECT_EQUAL <float>(dfVal1, dfVal, "The value at i = " + i.ToString() + " value = " + dfVal.ToString() + " does not equal value1 = " + dfVal1.ToString());
dfVal += dfInc;
}
dfVal = clrMap.GetValue(Color.FromArgb(0, 130, 124));
log.CHECK_NE(0, dfVal, "The value should not be zero.");
dfVal = clrMap.GetValue(Color.FromArgb(255, 252, 0));
log.CHECK_NE(0, dfVal, "The value should not be zero.");
log.WriteLine("DONE.");
}
public void TestQueryGeneralWAV()
{
PreTest.Init();
Log log = new Log("Test streaming database with general WAV data");
log.EnableTrace = true;
IXStreamDatabase db = new MyCaffeStreamDatabase(log);
try
{
string strSchema = "ConnectionCount=1;";
string strDataPath = getTestPath("\\MyCaffe\\test_data\\data\\wav", true);
string strParam = "FilePath=" + strDataPath + ";";
strParam = ParamPacker.Pack(strParam);
strSchema += "Connection0_CustomQueryName=StdWAVFileQuery;";
strSchema += "Connection0_CustomQueryParam=" + strParam + ";";
DateTime dt = DateTime.Today;
string strSettings = "";
db.Initialize(QUERY_TYPE.GENERAL, strSchema + strSettings);
int[] rgSize = db.QuerySize();
log.CHECK(rgSize != null, "The Query size should not be null.");
log.CHECK_EQ(rgSize.Length, 3, "The query size should have 3 items.");
log.CHECK_EQ(rgSize[0], 1, "The query size item 1 should be 1 for the number of files.");
log.CHECK_GE(rgSize[1], 2, "The query size item 0 (the channel count) should be greater than or equal to 1.");
log.CHECK_GE(rgSize[2], 1000, "The query size item 2 should be the number of samples in the query.");
int nH = rgSize[1];
int nW = rgSize[2];
int nCount = nH * nW;
Stopwatch sw = new Stopwatch();
sw.Start();
SimpleDatum sd = db.Query(int.MaxValue);
SimpleDatum sdEnd = db.Query(int.MaxValue);
sw.Stop();
double dfMs = sw.Elapsed.TotalMilliseconds;
log.WriteLine("Total Time = " + dfMs.ToString() + " ms.");
log.CHECK(sdEnd == null, "The last query should be null to show no more data exists.");
log.CHECK_EQ(sd.ItemCount, nCount, "There should be the same number in the data as the size[1] * size[2] returned by QuerySize.");
log.CHECK(sd.IsRealData, "The data should be real data, not byte.");
}
finally
{
db.Shutdown();
}
}
public void TestIndexQuery()
{
PreTest.Init();
Log log = new Log("Test Dataset Factory");
log.EnableTrace = true;
string strDs = "MNIST";
DatasetFactory factory = new DatasetFactory();
Stopwatch sw = new Stopwatch();
try
{
DatasetDescriptor ds = factory.LoadDataset(strDs);
factory.Open(ds.TrainingSource.ID);
sw.Start();
List <DbItem> rgItems = factory.LoadImageIndexes(false);
sw.Stop();
log.CHECK_EQ(rgItems.Count, ds.TrainingSource.ImageCount, "The query count should match the image count!");
factory.Close();
log.WriteLine("Query time = " + sw.Elapsed.TotalMilliseconds.ToString("N5") + " ms.");
sw.Restart();
int nMin = int.MaxValue;
int nMax = -int.MaxValue;
for (int i = 0; i < rgItems.Count; i++)
{
nMin = Math.Min(rgItems[i].Label, nMin);
nMax = Math.Max(rgItems[i].Label, nMax);
}
List <DbItem> rgBoosted = rgItems.Where(p => p.Boost > 0).ToList();
for (int nLabel = nMin; nLabel <= nMax; nLabel++)
{
List <DbItem> rgLabel = rgItems.Where(p => p.Label == nLabel).ToList();
}
sw.Stop();
log.WriteLine("Query time (profile) = " + sw.Elapsed.TotalMilliseconds.ToString("N5") + " ms.");
}
finally
{
factory.Dispose();
}
}
public void TestRandom2()
{
PreTest.Init();
Log log = new Log("Test CryptoRandom - 2 items");
log.EnableTrace = true;
int nCount = 2;
testRandom(log, CryptoRandom.METHOD.SYSTEM, nCount);
testRandom(log, CryptoRandom.METHOD.CRYPTO, nCount);
testRandomIdx(log, CryptoRandom.METHOD.UNIFORM_EXACT, nCount);
}
public Test(string strName, int nDeviceID = TestBase.DEFAULT_DEVICE_ID, EngineParameter.Engine engine = EngineParameter.Engine.DEFAULT, bool bHalf = false)
{
PreTest.Init();
m_dt = (typeof(T) == typeof(double)) ? DataType.DOUBLE : DataType.FLOAT;
m_cuda = GetCuda(nDeviceID);
m_bHalf = bHalf;
string str = name;
if (str.Length > 0)
{
str += " -> ";
}
m_log = GetLog(str + strName + " - type " + (typeof(T)).ToString());
m_engine = engine;
}
public void TestQuerySimpleTrippleStressTiming()
{
PreTest.Init();
Log log = new Log("Test streaming database");
log.EnableTrace = true;
IXStreamDatabase db = new MyCaffeStreamDatabase(log);
try
{
string strSchema = "ConnectionCount=3;";
string strParam = "Connection=TestCon;Table=TestTbl;Field=TestField;";
strParam = ParamPacker.Pack(strParam);
strSchema += "Connection0_CustomQueryName=Test1;";
strSchema += "Connection0_CustomQueryParam=" + strParam + ";";
strSchema += "Connection1_CustomQueryName=Test2;";
strSchema += "Connection1_CustomQueryParam=" + strParam + ";";
strSchema += "Connection2_CustomQueryName=Test3;";
strSchema += "Connection2_CustomQueryParam=" + strParam + ";";
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery1());
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery2());
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery3());
DateTime dt = DateTime.Today;
string strSettings = "QueryCount=5;Start=" + dt.ToShortDateString() + ";TimeSpanInMs=60000;SegmentSize=1;MaxCount=10;";
db.Initialize(QUERY_TYPE.SYNCHRONIZED, strSchema + strSettings);
int[] rgSize = db.QuerySize();
log.CHECK(rgSize != null, "The Query size should not be null.");
log.CHECK_EQ(rgSize.Length, 3, "The query size should have 3 items.");
log.CHECK_EQ(rgSize[0], 1, "The query size item 0 should be 1.");
log.CHECK_EQ(rgSize[1], 6, "The query size item 1 should be 3 for three fields (q1:sync,data; q2:data).");
log.CHECK_EQ(rgSize[2], 5, "The query size item 2 should be 5 for the number of items queried.");
int nDataIdx = 0;
int nH = rgSize[1];
int nW = rgSize[2];
int nCount = nH * nW;
List <SimpleDatum> rgSd = new List <SimpleDatum>();
Stopwatch sw = new Stopwatch();
int nIter = 10000;
sw.Start();
for (int i = 0; i < nIter; i++)
{
rgSd.Add(db.Query(int.MaxValue));
}
sw.Stop();
double dfMs = sw.Elapsed.TotalMilliseconds;
double dfMsPerQuery = dfMs / nIter;
log.WriteLine("Total Time = " + dfMs.ToString() + " ms, Ave time per query = " + dfMsPerQuery.ToString() + " ms.");
for (int i = 0; i < rgSd.Count; i++)
{
SimpleDatum sd = rgSd[i];
log.CHECK(sd != null, "The SD returned should not be null.");
log.CHECK_EQ(sd.ItemCount, nCount, "There should be " + rgSize[1].ToString() + "x" + rgSize[2].ToString() + " items in the data.");
log.CHECK_EQ(sd.Channels, rgSize[0], "The channels are not as expected.");
log.CHECK_EQ(sd.Height, rgSize[1], "The height is not as expected.");
log.CHECK_EQ(sd.Width, rgSize[2], "The width is not as expected.");
for (int j = 0; j < nW; j++)
{
DateTime dt1 = Utility.ConvertTimeFromMinutes(sd.GetDataAtD(j));
log.CHECK(dt1 == dt, "The time sync is incorrect.");
dt += TimeSpan.FromMinutes(1);
double df1 = sd.GetDataAtD((nW * 1) + j);
int nVal1 = (int)df1;
log.CHECK_EQ(nVal1, nDataIdx, "The data value is incorrect.");
double df2 = sd.GetDataAtD((nW * 2) + j);
int nVal2 = (int)df2;
log.CHECK_EQ(nVal2, nDataIdx, "The data value is incorrect.");
double df3 = sd.GetDataAtD((nW * 3) + j);
int nVal3 = (int)df3;
log.CHECK_EQ(nVal3, nDataIdx, "The data value is incorrect.");
double df4 = sd.GetDataAtD((nW * 4) + j);
int nVal4 = (int)df4;
log.CHECK_EQ(nVal4, nDataIdx, "The data value is incorrect.");
double df5 = sd.GetDataAtD((nW * 5) + j);
int nVal5 = (int)df5;
log.CHECK_EQ(nVal5, nDataIdx, "The data value is incorrect.");
nDataIdx++;
}
}
}
finally
{
db.Shutdown();
}
}
public void TestQuerySimple()
{
PreTest.Init();
Log log = new Log("Test streaming database");
log.EnableTrace = true;
IXStreamDatabase db = new MyCaffeStreamDatabase(log);
try
{
string strSchema = "ConnectionCount=1;";
string strParam = "Connection=TestCon;Table=TestTbl;Field=TestField;";
strParam = ParamPacker.Pack(strParam);
strSchema += "Connection0_CustomQueryName=Test1;";
strSchema += "Connection0_CustomQueryParam=" + strParam + ";";
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery1());
DateTime dt = DateTime.Today;
string strSettings = "QueryCount=5;Start=" + DateTime.Today.ToShortDateString() + ";TimeSpanInMs=60000;SegmentSize=1;MaxCount=10;";
db.Initialize(QUERY_TYPE.SYNCHRONIZED, strSchema + strSettings);
int[] rgSize = db.QuerySize();
log.CHECK(rgSize != null, "The Query size should not be null.");
log.CHECK_EQ(rgSize.Length, 3, "The query size should have 2 items.");
log.CHECK_EQ(rgSize[0], 1, "The query size item 0 should be 1.");
log.CHECK_EQ(rgSize[1], 2, "The query size item 1 should be 2 for two fields.");
log.CHECK_EQ(rgSize[2], 5, "The query size item 2 should be 5 for the number of items queried.");
SimpleDatum sd;
int nDataIdx = 0;
int nH = rgSize[1];
int nW = rgSize[2];
int nCount = nH * nW;
for (int i = 0; i < 2; i++)
{
sd = db.Query(int.MaxValue);
log.CHECK(sd != null, "The SD returned should not be null.");
log.CHECK_EQ(sd.ItemCount, nCount, "There should be " + rgSize[1].ToString() + "x" + rgSize[2].ToString() + " items in the data.");
log.CHECK_EQ(sd.Channels, rgSize[0], "The channels are not as expected.");
log.CHECK_EQ(sd.Height, rgSize[1], "The height is not as expected.");
log.CHECK_EQ(sd.Width, rgSize[2], "The width is not as expected.");
for (int j = 0; j < nW; j++)
{
DateTime dt1 = Utility.ConvertTimeFromMinutes(sd.GetDataAtD(j));
log.CHECK(dt1 == dt, "The time sync is incorrect.");
dt += TimeSpan.FromMinutes(1);
double df = sd.GetDataAtD(nW + j);
int nVal = (int)df;
log.CHECK_EQ(nVal, nDataIdx, "The data value is incorrect.");
nDataIdx++;
}
}
}
finally
{
db.Shutdown();
}
}
public void TestQuerySimpleDualDifferentEndAndReset()
{
PreTest.Init();
Log log = new Log("Test streaming database");
log.EnableTrace = true;
IXStreamDatabase db = new MyCaffeStreamDatabase(log);
try
{
string strSchema = "ConnectionCount=2;";
string strParam1 = "Connection=TestCon;Table=TestTbl;Field=TestField;EndIdx=10;";
string strParam2 = "Connection=TestCon;Table=TestTbl;Field=TestField;EndIdx=15;";
strParam1 = ParamPacker.Pack(strParam1);
strParam2 = ParamPacker.Pack(strParam2);
strSchema += "Connection0_CustomQueryName=Test1;";
strSchema += "Connection0_CustomQueryParam=" + strParam1 + ";";
strSchema += "Connection1_CustomQueryName=Test2;";
strSchema += "Connection1_CustomQueryParam=" + strParam2 + ";";
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery1());
((MyCaffeStreamDatabase)db).AddDirectQuery(new CustomQuery2());
DateTime dt = DateTime.Today;
string strSettings = "QueryCount=5;Start=" + dt.ToShortDateString() + ";TimeSpanInMs=60000;SegmentSize=1;MaxCount=10;";
db.Initialize(QUERY_TYPE.SYNCHRONIZED, strSchema + strSettings);
int[] rgSize = db.QuerySize();
log.CHECK(rgSize != null, "The Query size should not be null.");
log.CHECK_EQ(rgSize.Length, 3, "The query size should have 3 items.");
log.CHECK_EQ(rgSize[0], 1, "The query size item 0 should be 1.");
log.CHECK_EQ(rgSize[1], 3, "The query size item 1 should be 3 for three fields (q1:sync,data; q2:data).");
log.CHECK_EQ(rgSize[2], 5, "The query size item 2 should be 5 for the number of items queried.");
SimpleDatum sd;
int nDataIdx = 0;
int nH = rgSize[1];
int nW = rgSize[2];
int nCount = nH * nW;
for (int i = 0; i < 6; i++)
{
sd = db.Query(int.MaxValue);
if (i == 0 || i == 1 || i == 3 || i == 4)
{
log.CHECK(sd != null, "The SD returned should not be null.");
log.CHECK_EQ(sd.ItemCount, nCount, "There should be " + rgSize[1].ToString() + "x" + rgSize[2].ToString() + " items in the data.");
log.CHECK_EQ(sd.Channels, rgSize[0], "The channels are not as expected.");
log.CHECK_EQ(sd.Height, rgSize[1], "The height is not as expected.");
log.CHECK_EQ(sd.Width, rgSize[2], "The width is not as expected.");
for (int j = 0; j < nW; j++)
{
DateTime dt1 = Utility.ConvertTimeFromMinutes(sd.GetDataAtD(j));
log.CHECK(dt1 == dt, "The time sync is incorrect.");
dt += TimeSpan.FromMinutes(1);
double df1 = sd.GetDataAtD((nW * 1) + j);
int nVal1 = (int)df1;
log.CHECK_EQ(nVal1, nDataIdx, "The data value is incorrect.");
double df2 = sd.GetDataAtD((nW * 2) + j);
int nVal2 = (int)df2;
log.CHECK_EQ(nVal2, nDataIdx, "The data value is incorrect.");
nDataIdx++;
}
}
else
{
log.CHECK(sd == null, "Since we are past the end, the sd should be null.");
dt = DateTime.Today;
nDataIdx = 0;
db.Reset(0);
}
}
}
finally
{
db.Shutdown();
}
}
private void testRandomIdx(Log log, CryptoRandom.METHOD method, int nBuckets)
{
PreTest.Init();
BucketCollection col = new BucketCollection(0.0, 1.0, nBuckets);
CryptoRandom rand = new CryptoRandom(method, Guid.NewGuid().GetHashCode());
int nTotal = 100000;
log.WriteLine("Testing (" + nBuckets.ToString() + ") " + method.ToString());
List <int> rgIdx1 = new List <int>();
List <List <int> > rgrgPermutations = new List <List <int> >();
for (int i = 0; i < nTotal / nBuckets; i++)
{
List <int> rgPermutation = new List <int>();
for (int j = 0; j < nBuckets; j++)
{
int nIdx = rand.Next(nBuckets);
double dfPct = (double)nIdx / (double)nBuckets;
rgPermutation.Add(nIdx);
col.Add(dfPct);
}
rgrgPermutations.Add(rgPermutation);
}
string str = "";
List <double> rgdf = new List <double>();
for (int i = 0; i < nBuckets; i++)
{
double dfPct = col[i].Count / (double)nTotal;
str += dfPct.ToString("P");
str += ", ";
rgdf.Add(dfPct);
}
str = str.TrimEnd(',', ' ');
log.WriteLine(method.ToString() + " =>> " + str);
double dfStdev = stdDev(rgdf, false);
log.WriteLine(method.ToString() + " stdev = " + dfStdev.ToString());
// Verify permuation uniqueness
int nDuplicateCount = 0;
int nPermutationCount = rgrgPermutations.Count;
Stopwatch sw = new Stopwatch();
sw.Start();
int nProgressIdx = 0;
while (rgrgPermutations.Count > 1)
{
List <int> rgPermutation1 = rgrgPermutations[0];
rgrgPermutations.RemoveAt(0);
List <int> rgRemove = new List <int>();
for (int j = 0; j < rgrgPermutations.Count; j++)
{
if (compareLists(rgPermutation1, rgrgPermutations[j]))
{
nDuplicateCount++;
rgRemove.Add(j);
}
}
for (int j = rgRemove.Count - 1; j >= 0; j--)
{
rgrgPermutations.RemoveAt(rgRemove[j]);
}
if (sw.Elapsed.TotalMilliseconds > 2000)
{
log.Progress = (double)nProgressIdx / (double)nPermutationCount;
log.WriteLine("Permutation checking at " + log.Progress.ToString("P") + "...");
sw.Restart();
}
nProgressIdx++;
}
log.WriteLine("Out of " + nPermutationCount.ToString("N0") + " permutations, " + nDuplicateCount.ToString("N0") + " duplicates were found (" + ((double)nDuplicateCount / nPermutationCount).ToString("P") + ").");
}