public static void RSelectSpillageTest2()
{
string guid = Guid.NewGuid().ToString().Replace("-", "");
string tablename = "rselect_test_" + guid;
string indexname = guid + "apple";
string indexname_order = guid + "order";
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider");
{
Console.WriteLine("Creating data, table, and index...");
string job = (@"<SourceCode>
<Jobs>
<Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
//Clean up previous data.
Shell(@`Qizmt del data_Input" + guid + @"*`);
Shell(@`Qizmt del data_Output" + guid + @"*`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
try
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `drop table " + tablename + @"`;
cmd.ExecuteNonQuery();
cmd.CommandText = `drop rindex " + indexname + @"`;
cmd.ExecuteNonQuery();
conn.Close();
}
catch
{
}
}
}
]]>
</Local>
</Job>
<Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
dfsoutput.WriteLine(`1`);
}
]]>
</Remote>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
for(int i = 0; i < 2; i++)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutInt(0);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
for(int i = 0; i < 2; i++)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutInt(0);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
for(int i = 0; i < 2; i++)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutInt(i);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"4.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
for(int i = 0; i < 2; i++)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutInt(i);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"5.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
for(int i = 0; i < 2; i++)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
rkey.PutInt(i);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>RDBMS_DBCORE</Using>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin data_Output" + guid + @"4.bin data_Output" + guid + @"5.bin +data_Output" + guid + @".bin`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create table " + tablename + @" (id int, rid int)`;
cmd.ExecuteNonQuery();
cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`;
cmd.ExecuteNonQuery();
conn.Close();
}
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH ON id`;
cmd.ExecuteNonQuery();
cmd.CommandText = `create rindex " + indexname_order + @" from " + tablename + @" pinmemoryHASH keepvalueorder ON id`;
cmd.ExecuteNonQuery();
conn.Close();
}
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>").Replace('`', '"');
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", "");
if (System.IO.Directory.Exists(tempdir))
{
System.IO.Directory.Delete(tempdir, true);
}
System.IO.Directory.CreateDirectory(tempdir);
string tempjobname = Guid.NewGuid().ToString();
System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job);
Exec.Shell("Qizmt importdir \"" + tempdir + "\"");
Exec.Shell("dspace exec " + tempjobname);
//Clean up
Exec.Shell(@"Qizmt del " + tempjobname);
System.IO.Directory.Delete(tempdir, true);
}
{
Console.WriteLine("RSelecting from rindex...");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = localhost; rindex=pooled";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "rselect * from " + indexname + " where key = 2";
DbDataReader reader = cmd.ExecuteReader();
int cnt = 0;
while (reader.Read())
{
if (reader.GetInt32(0) != 2)
{
throw new Exception("Expected value.");
}
cnt++;
}
reader.Close();
if (cnt != 6)
{
throw new Exception("Expected 6 rows, but received only " + cnt.ToString());
}
conn.Close();
}
{
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = localhost";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "drop table " + tablename;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop rindex " + indexname;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop rindex " + indexname_order;
cmd.ExecuteNonQuery();
conn.Close();
}
}
public static void RIndexOutlierDelete()
{
string guid = Guid.NewGuid().ToString().Replace("-", "");
string tablename = "rselect_test_" + guid;
string indexname = guid + "apple";
string indexname_order = guid + "order";
string[] allhosts = null;
{
string[] installs = Exec.Shell("Qizmt slaveinstalls").Trim()
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
allhosts = new string[installs.Length];
for (int ip = 0; ip < installs.Length; ip++)
{
string[] parts = installs[ip].Split(' ');
allhosts[ip] = parts[0];
}
}
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider");
{
Console.WriteLine("Creating data, table, and index...");
string job = (@"<SourceCode>
<Jobs>
<Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
//Clean up previous data.
Shell(@`Qizmt del data_Input" + guid + @"*`);
Shell(@`Qizmt del data_Output" + guid + @"*`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
try
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `drop table " + tablename + @"`;
cmd.ExecuteNonQuery();
cmd.CommandText = `drop rindex " + indexname + @"`;
cmd.ExecuteNonQuery();
conn.Close();
}
catch
{
}
}
}
]]>
</Local>
</Job>
<Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
dfsoutput.WriteLine(`1`);
}
]]>
</Remote>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(-2);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(-2);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(-2);
rkey.PutInt(3);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(5);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(5);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(5);
rkey.PutInt(3);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(5);
rkey.PutInt(4);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(10);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(10);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(10);
rkey.PutInt(3);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(19);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(19);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(20);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(20);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(20);
rkey.PutInt(3);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(20);
rkey.PutInt(4);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(20);
rkey.PutInt(5);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(20);
rkey.PutInt(6);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(21);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(21);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(3);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(4);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(5);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(6);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(30);
rkey.PutInt(7);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>RDBMS_DBCORE</Using>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin +data_Output" + guid + @".bin`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create table " + tablename + @" (id int, rid int)`;
cmd.ExecuteNonQuery();
cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`;
cmd.ExecuteNonQuery();
conn.Close();
}
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH OUTLIER Delete 3 ON id`;
cmd.ExecuteNonQuery();
conn.Close();
}
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>").Replace('`', '"');
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", "");
if (System.IO.Directory.Exists(tempdir))
{
System.IO.Directory.Delete(tempdir, true);
}
System.IO.Directory.CreateDirectory(tempdir);
string tempjobname = Guid.NewGuid().ToString();
System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job);
Exec.Shell("Qizmt importdir \"" + tempdir + "\"");
Exec.Shell("dspace exec " + tempjobname);
//Clean up
Exec.Shell(@"Qizmt del " + tempjobname);
System.IO.Directory.Delete(tempdir, true);
}
{
Console.WriteLine("RInsert/RDeleting rindex...");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText =
"rinsert into " + indexname + " values(22,1) where key = 22\0" +
"rinsert into " + indexname + " values(22,2) where key = 22\0" +
"rinsert into " + indexname + " values(22,3) where key = 22\0" +
"rinsert into " + indexname + " values(22,4) where key = 22\0" +
"rinsert into " + indexname + " values(21,3) where key = 21\0" +
"rinsert into " + indexname + " values(21,4) where key = 21\0" +
"rinsert into " + indexname + " values(21,5) where key = 21\0" +
"rdelete from " + indexname + " where key = 21 and rid = 4\0" +
"rdelete from " + indexname + " where key = 21 and rid = 5\0" +
"rinsert into " + indexname + " values(8,1) where key = 8\0" +
"rinsert into " + indexname + " values(8,2) where key = 8\0" +
"rinsert into " + indexname + " values(8,3) where key = 8\0" +
"rinsert into " + indexname + " values(8,4) where key = 8\0" +
"rinsert into " + indexname + " values(6,1) where key = 6\0" +
"rinsert into " + indexname + " values(6,2) where key = 6\0" +
"rinsert into " + indexname + " values(5,5) where key = 5\0" +
"rinsert into " + indexname + " values(5,6) where key = 5\0" +
"rinsert into " + indexname + " values(-2,4) where key = -2\0";
cmd.ExecuteNonQuery();
conn.Close();
}
{
Console.WriteLine("RSelecting from rindex...");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "rselect * from " + indexname + " where key = -2 or key = 5 or key = 6 or key = 8 or key= 10 or key = 19 or key=20 or key = 21 or key = 22 or key=30";
DbDataReader reader = cmd.ExecuteReader();
int cnt = 0;
List <KeyValuePair <int, int> > results = new List <KeyValuePair <int, int> >();
while (reader.Read())
{
int x = reader.GetInt32(0);
int y = reader.GetInt32(1);
KeyValuePair <int, int> row = new KeyValuePair <int, int>(x, y);
results.Add(row);
}
reader.Close();
conn.Close();
results.Sort(delegate(KeyValuePair <int, int> x, KeyValuePair <int, int> y)
{
if (x.Key != y.Key)
{
return(x.Key.CompareTo(y.Key));
}
else
{
return(x.Value.CompareTo(y.Value));
}
});
List <KeyValuePair <int, int> > expected = new List <KeyValuePair <int, int> >();
expected.Add(new KeyValuePair <int, int>(5, 1));
expected.Add(new KeyValuePair <int, int>(5, 2));
expected.Add(new KeyValuePair <int, int>(6, 1));
expected.Add(new KeyValuePair <int, int>(6, 2));
expected.Add(new KeyValuePair <int, int>(10, 1));
expected.Add(new KeyValuePair <int, int>(10, 2));
expected.Add(new KeyValuePair <int, int>(10, 3));
expected.Add(new KeyValuePair <int, int>(19, 1));
expected.Add(new KeyValuePair <int, int>(19, 2));
expected.Add(new KeyValuePair <int, int>(21, 1));
expected.Add(new KeyValuePair <int, int>(21, 2));
expected.Add(new KeyValuePair <int, int>(21, 3));
if (results.Count != expected.Count)
{
throw new Exception("Expected " + expected.Count.ToString() + " rows to be returned. But received " + results.Count.ToString() + " rows instead.");
}
for (int i = 0; i < results.Count; i++)
{
if (results[i].Key != expected[i].Key || results[i].Value != expected[i].Value)
{
throw new Exception("Row returned is different from expected results. Received row:(" + results[i].Key.ToString() + "," + results[i].Value.ToString() + "). Expected row:(" + expected[i].Key.ToString() + "," + expected[i].Value.ToString() + ")");
}
}
}
{
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = localhost";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "drop table " + tablename;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop rindex " + indexname;
cmd.ExecuteNonQuery();
conn.Close();
}
}
public static void FaultTolerantExecution()
{
if (!MySpace.DataMining.AELight.FTTest.enabled)
{
throw new Exception("TESTFAULTTOLERANT is not #defined. Need Qizmt build with all #define TESTFAULTTOLERANT uncommented.");
}
string guid = "A275169D14B34df48229FC3F43A0AA31";
string tblUsers = "users_" + guid;
string tblPageviews = "pageviews_" + guid;
string tblPageToNames = "pagetonames_" + guid;
string controlfile = @"\\" + MySpace.DataMining.AELight.Surrogate.MasterHost + @"\c$\temp\"
+ MySpace.DataMining.AELight.FTTest.controlfilename;
string tempdir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) +
@"\" + Guid.NewGuid().ToString();
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("Qizmt_DataProvider");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = localhost; mr bypass=10; fault tolerant execution=enabled";
DbCommand cmd = conn.CreateCommand();
try
{
{
System.IO.Directory.CreateDirectory(tempdir);
#region genPageViews
System.IO.File.WriteAllText(tempdir + @"\genPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml",
@"<SourceCode>
<Jobs>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del tblPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.bin`);
}
]]>
</Local>
</Job>
<Job Name=`CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://tblPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.bin@8</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
List<byte> buf = new List<byte>(24);
{
recordset rs = recordset.Prepare();
rs.PutInt(100);
rs.PutInt(1);
rs.ToByteSlice().AppendTo(buf);
dfsoutput.WriteRecord(buf);
}
{
buf.Clear();
recordset rs = recordset.Prepare();
rs.PutInt(101);
rs.PutInt(1);
rs.ToByteSlice().AppendTo(buf);
dfsoutput.WriteRecord(buf);
}
{
buf.Clear();
recordset rs = recordset.Prepare();
rs.PutInt(300);
rs.PutInt(3);
rs.ToByteSlice().AppendTo(buf);
dfsoutput.WriteRecord(buf);
}
{
buf.Clear();
recordset rs = recordset.Prepare();
rs.PutInt(301);
rs.PutInt(3);
rs.ToByteSlice().AppendTo(buf);
dfsoutput.WriteRecord(buf);
}
{
buf.Clear();
recordset rs = recordset.Prepare();
rs.PutInt(302);
rs.PutInt(3);
rs.ToByteSlice().AppendTo(buf);
dfsoutput.WriteRecord(buf);
}
{
buf.Clear();
recordset rs = recordset.Prepare();
rs.PutInt(400);
rs.PutInt(4);
rs.ToByteSlice().AppendTo(buf);
dfsoutput.WriteRecord(buf);
}
}
]]>
</Remote>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"'));
#endregion
#region genPageToNames
System.IO.File.WriteAllText(tempdir + @"\genPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml",
@"<SourceCode>
<Jobs>
<Job Name=`Preprocessing` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt del tblPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.txt`);
}
]]>
</Local>
</Job>
<Job Name=`CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://tblPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
//Create sample data.
dfsoutput.WriteLine(`90,a`);
dfsoutput.WriteLine(`91,b`);
dfsoutput.WriteLine(`92,c`);
dfsoutput.WriteLine(`93,d`);
dfsoutput.WriteLine(`94,e`);
}
]]>
</Remote>
</Job>
</Jobs>
</SourceCode>
".Replace('`', '"'));
#endregion
Exec.Shell("qizmt importdir \"" + tempdir + "\"");
Exec.Shell("qizmt exec genPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml");
Exec.Shell("qizmt exec genPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml");
}
conn.Open();
#region createtables
{
Console.WriteLine("Creating tables...");
try
{
cmd.CommandText = "drop table " + tblUsers;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop table " + tblPageviews;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop table " + tblPageToNames;
cmd.ExecuteNonQuery();
conn.Close(); //flush
}
catch
{
}
conn.Open();
cmd.CommandText = "create table " + tblUsers + " (userid int, username char(50))";
cmd.ExecuteNonQuery();
cmd.CommandText = "insert into " + tblUsers + " values (1, 'john')";
cmd.ExecuteNonQuery();
cmd.CommandText = "insert into " + tblUsers + " values (2, 'mary')";
cmd.ExecuteNonQuery();
cmd.CommandText = "insert into " + tblUsers + " values (3, 'joe')";
cmd.ExecuteNonQuery();
cmd.CommandText = "create table " + tblPageviews + " (pageid int, uid int)";
cmd.ExecuteNonQuery();
cmd.CommandText = "create table " + tblPageToNames + " (pageid int, username char(50))";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
}
#endregion
#region testIMPORT
{
string phase = "exchangeowned";
Console.WriteLine("Testing IMPORT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "INSERT INTO " + tblPageviews + " IMPORT 'tblPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.bin'";
cmd.ExecuteNonQuery();
conn.Close();
conn.Open();
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "select * from " + tblPageviews;
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
int userid = reader.GetInt32(1);
if (!(pageid == 100 && userid == 1) && !(pageid == 101 && userid == 1) &&
!(pageid == 300 && userid == 3) && !(pageid == 301 && userid == 3) && !(pageid == 302 && userid == 3) &&
!(pageid == 400 && userid == 4))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 6)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testIMPORTLINES
{
string phase = "reduce";
Console.WriteLine("Testing IMPORTLINES in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "INSERT INTO " + tblPageToNames + " IMPORTLINES 'tblPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.txt'";
cmd.ExecuteNonQuery();
conn.Close();
conn.Open();
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "select * from " + tblPageToNames;
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
string name = reader.GetString(1);
if (!(pageid == 90 && name == "a") && !(pageid == 91 && name == "b") &&
!(pageid == 92 && name == "c") && !(pageid == 93 && name == "d") &&
!(pageid == 94 && name == "e"))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 5)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testSELECT
{
string phase = "map";
Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "select * from " + tblUsers + " where userid = 2";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
if (reader.GetString(1) != "mary")
{
throw new Exception("Unexpected value for field username");
}
}
reader.Close();
if (count != 1)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testSELECT
{
string phase = "exchangeremote";
Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "select * from " + tblUsers + " where userid > 0";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
string username = reader.GetString(1);
if (username != "mary" && username != "john" && username != "joe")
{
throw new Exception("Unexpected value for field username");
}
}
reader.Close();
if (count != 3)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testSELECT
{
string phase = "exchangeowned";
Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "select username from " + tblUsers + " where userid=3";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
if (reader.GetString(0) != "joe")
{
throw new Exception("Unexpected value for field username");
}
}
reader.Close();
if (count != 1)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testSELECT
{
string phase = "sort";
Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "select username from " + tblUsers + " where abs(userid)=3";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
if (reader.GetString(0) != "joe")
{
throw new Exception("Unexpected value for field username");
}
}
reader.Close();
if (count != 1)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testSELECT
{
Console.WriteLine("Testing SELECT in FTE mode with no failure...");
System.IO.File.WriteAllText(controlfile, ""); //Do not simulate failure since FTE is not enabled for SELECT func(column)
cmd.CommandText = "select max(pageid), uid from " + tblPageviews + " group by uid";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
int uid = reader.GetInt32(1);
if (!(pageid == 101 && uid == 1) && !(pageid == 302 && uid == 3) && !(pageid == 400 && uid == 4))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 3)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testINNERJOIN
{
string phase = "reduce";
Console.WriteLine("Testing INNER JOIN in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "SELECT * FROM " + tblPageviews + " inner join " + tblUsers + " on uid=userid";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
string username = reader.GetString(3);
if (!(pageid == 100 && username == "john") && !(pageid == 101 && username == "john") &&
!(pageid == 300 && username == "joe") && !(pageid == 301 && username == "joe") &&
!(pageid == 302 && username == "joe"))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 5)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testINNERJOIN
{
string phase = "sort";
Console.WriteLine("Testing INNER JOIN in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
long jid = GetJobID("RDBMS_JoinOn.DBCORE");
if (jid != -1)
{
throw new Exception("Expected jobid = -1");
}
List <string> errors = new List <string>();
System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
while (jid == -1)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
jid = GetJobID("RDBMS_JoinOn.DBCORE");
}
Console.WriteLine("jid={0}", jid);
for (; ;)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
if (GetJobOutputIndexOf(jid, "Replicating") > -1)
{
//Can clear control file for the next job run.
ClearControlFile(controlfile);
Console.WriteLine("control file cleared");
break;
}
}
}));
tm.IsBackground = true;
tm.Start();
System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
cmd.CommandText = "select max(pageid), username from " + tblPageviews +
" inner join " + tblUsers + " on uid = userid group by username";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
string username = reader.GetString(1);
if (!(pageid == 101 && username == "john") && !(pageid == 302 && username == "joe"))
{
errors.Add("Unexpected value");
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 2)
{
errors.Add("Unexpected rows returned");
throw new Exception("Unexpected rows returned");
}
}));
ts.IsBackground = true;
ts.Start();
tm.Join();
ts.Join();
if (errors.Count > 0)
{
StringBuilder sb = new StringBuilder();
foreach (string err in errors)
{
sb.Append(err);
sb.Append(Environment.NewLine);
}
throw new Exception("Error during job run:" + sb.ToString());
}
Console.WriteLine("Done");
}
#endregion
#region testLEFTJOIN
{
string phase = "replication";
Console.WriteLine("Testing LEFT OUTER JOIN in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "SELECT * FROM " + tblPageviews + " left outer join " + tblUsers + " on uid=userid";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = (int)reader[0];
string username = DBNull.Value.Equals(reader[3]) ? null : (string)reader[3];
if (!(pageid == 100 && username == "john") && !(pageid == 101 && username == "john") &&
!(pageid == 300 && username == "joe") && !(pageid == 301 && username == "joe") &&
!(pageid == 302 && username == "joe") && !(pageid == 400 && username == null))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 6)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testRIGHTJOIN
{
string phase = "map";
Console.WriteLine("Testing RIGHT OUTER JOIN in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "SELECT * FROM " + tblPageviews + " right outer join " + tblUsers + " on uid=userid";
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int userid = (int)reader[2];
int pageid = DBNull.Value.Equals(reader[0]) ? -1 : (int)reader[0];
if (!(userid == 1 && pageid == 100) && !(userid == 1 && pageid == 101) &&
!(userid == 3 && pageid == 300) && !(userid == 3 && pageid == 301) &&
!(userid == 3 && pageid == 302) && !(userid == 2 && pageid == -1))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 6)
{
throw new Exception("Unexpected rows returned");
}
Console.WriteLine("Done");
}
#endregion
#region testDELETE
{
{
string phase = "exchangeremote";
Console.WriteLine("Testing DELETE in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "DELETE FROM " + tblPageToNames + " where username='******'";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
Console.WriteLine("+");
}
{
string phase = "exchangeowned";
Console.WriteLine("Testing DELETE in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "DELETE FROM " + tblPageToNames + " where abs(pageid)= 93";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
Console.WriteLine("+");
}
{
string phase = "sort";
Console.WriteLine("Testing DELETE in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "delete from " + tblPageToNames + " where mod(pageid, 2) = 0";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
Console.WriteLine("+");
}
{
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "select * from " + tblPageToNames;
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
}
reader.Close();
if (count != 0)
{
throw new Exception("Unexpected number of rows returned");
}
}
Console.WriteLine("Done");
}
#endregion
#region testINSERT
{
{
string phase = "reduce";
Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
cmd.CommandText = "insert into " + tblPageToNames +
" select userid, username from " + tblUsers + " where userid > 2";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
Console.WriteLine("+");
}
{
Console.WriteLine("Testing INSERT in FTE mode with no failure");
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "insert into " + tblPageToNames +
" select abs(userid), username from " + tblUsers + " where userid = 2";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
Console.WriteLine("+");
}
{
Console.WriteLine("Testing INSERT in FTE mode with no failure");
cmd.CommandText = "insert into " + tblPageToNames +
" select userid, username from " + tblUsers +
" where userid = 1 order by userid";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
Console.WriteLine("+");
}
{
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "select * from " + tblPageToNames;
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
string username = reader.GetString(1);
if (!(pageid == 1 && username == "john") && !(pageid == 2 && username == "mary") &&
!(pageid == 3 && username == "joe"))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 3)
{
throw new Exception("Unexpected number of rows returned");
}
}
Console.WriteLine("Done");
}
#endregion
#region testINSERT
{
//prepare
{
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "delete from " + tblPageToNames + " where pageid > 0";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
}
{
string phase = "exchangeowned";
Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
long jid = GetJobID("RDBMS_JoinOn.DBCORE");
if (jid != -1)
{
throw new Exception("Expected jobid = -1");
}
System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
while (jid == -1)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
jid = GetJobID("RDBMS_JoinOn.DBCORE");
}
Console.WriteLine("jid={0}", jid);
for (; ;)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
if (GetJobOutputIndexOf(jid, "Replicating") > -1)
{
//Can clear control file for the next job run.
ClearControlFile(controlfile);
Console.WriteLine("control file cleared");
break;
}
}
}));
tm.IsBackground = true;
tm.Start();
System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
cmd.CommandText = "insert into " + tblPageToNames +
" select abs(pageid), username from " + tblPageviews +
" inner join " + tblUsers + " on uid = userid where abs(userid) = 1";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
}));
ts.IsBackground = true;
ts.Start();
tm.Join();
ts.Join();
Console.WriteLine("+");
}
{
string phase = "exchangeremote";
Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
long jid = GetJobID("RDBMS_JoinOn.DBCORE");
if (jid != -1)
{
throw new Exception("Expected jobid = -1");
}
System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
while (jid == -1)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
jid = GetJobID("RDBMS_JoinOn.DBCORE");
}
Console.WriteLine("jid={0}", jid);
for (; ;)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
if (GetJobOutputIndexOf(jid, "Replicating") > -1)
{
//Can clear control file for the next job run.
ClearControlFile(controlfile);
Console.WriteLine("control file cleared");
break;
}
}
}));
tm.IsBackground = true;
tm.Start();
System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
cmd.CommandText = "insert into " + tblPageToNames +
" select abs(pageid), username from " + tblPageviews +
" inner join " + tblUsers +
" on uid = userid where abs(userid) = 3 order by pageid";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
}));
ts.IsBackground = true;
ts.Start();
tm.Join();
ts.Join();
Console.WriteLine("+");
}
{
string phase = "sort";
Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase);
using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile))
{
w.WriteLine("{1}:" + phase);
}
long jid = GetJobID("RDBMS_JoinOn.DBCORE");
if (jid != -1)
{
throw new Exception("Expected jobid = -1");
}
System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
while (jid == -1)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
jid = GetJobID("RDBMS_JoinOn.DBCORE");
}
Console.WriteLine("jid={0}", jid);
for (; ;)
{
Console.Write(".");
System.Threading.Thread.Sleep(1000);
if (GetJobOutputIndexOf(jid, "Replicating") > -1)
{
//Can clear control file for the next job run.
ClearControlFile(controlfile);
Console.WriteLine("control file cleared");
break;
}
}
}));
tm.IsBackground = true;
tm.Start();
System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate()
{
cmd.CommandText = "insert into " + tblPageToNames +
" select max(userid), username from " + tblPageviews +
" inner join " + tblUsers +
" on uid = userid group by username";
cmd.ExecuteNonQuery();
conn.Close(); //flush
conn.Open();
}));
ts.IsBackground = true;
ts.Start();
tm.Join();
ts.Join();
Console.WriteLine("+");
}
{
System.IO.File.WriteAllText(controlfile, "");
cmd.CommandText = "select * from " + tblPageToNames;
DbDataReader reader = cmd.ExecuteReader();
int count = 0;
while (reader.Read())
{
count++;
int pageid = reader.GetInt32(0);
string username = reader.GetString(1);
if (!(pageid == 100 && username == "john") && !(pageid == 101 && username == "john") &&
!(pageid == 300 && username == "joe") && !(pageid == 301 && username == "joe") &&
!(pageid == 302 && username == "joe") && !(pageid == 1 && username == "john") &&
!(pageid == 3 && username == "joe"))
{
throw new Exception("Unexpected value");
}
}
reader.Close();
if (count != 7)
{
throw new Exception("Unexpected number of rows returned");
}
}
Console.WriteLine("Done");
}
#endregion
#region cleanup
try
{
cmd.CommandText = "drop table " + tblUsers;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop table " + tblPageviews;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop table " + tblPageToNames;
cmd.ExecuteNonQuery();
}
catch
{
}
#endregion
}
finally
{
if (conn.State == ConnectionState.Open)
{
try
{
conn.Close();
}
catch (Exception e)
{
Console.WriteLine("Closing error: " + e.ToString());
}
}
System.IO.Directory.Delete(tempdir, true);
Exec.Shell("qizmt del *873200A2-EFD4-4136-9209-A807CF8BA3C2*");
System.IO.File.Delete(controlfile);
}
}
static int GetJobOutputIndexOf(long jobid, string token)
{
string output = Exec.Shell("qizmt viewjob " + jobid.ToString());
return(output.IndexOf(token, StringComparison.OrdinalIgnoreCase));
}
public static void RIndexReplication()
{
{
int replicationFactor = 0;
string[] lines = Exec.Shell("Qizmt replicationview").Split('\n');
for (int i = 0; i < lines.Length; i++)
{
string line = lines[i].Trim();
if (line.Length > 0)
{
int del = line.LastIndexOf(' ');
replicationFactor = Int32.Parse(line.Substring(del + 1));
if (replicationFactor > 0)
{
break;
}
}
}
if (replicationFactor < 2)
{
throw new Exception("Cannot run RIndexReplication regression test when replication factor is less than 2.");
}
}
string guid = Guid.NewGuid().ToString().Replace("-", "");
string tablename = "rselect_test_" + guid;
string indexname = guid + "apple";
string indexname_order = guid + "order";
string[] allhosts = null;
string qizmtrootdir = "";
{
string[] installs = Exec.Shell("Qizmt slaveinstalls").Trim()
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
allhosts = new string[installs.Length];
for (int ip = 0; ip < installs.Length; ip++)
{
string[] parts = installs[ip].Split(' ');
string installpath = parts[1];
int del = installpath.IndexOf(@"\", 2);
qizmtrootdir = installpath.Substring(del + 1);
allhosts[ip] = parts[0];
}
}
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider");
{
Console.WriteLine("Creating data, table, and index...");
string job = (@"<SourceCode>
<Jobs>
<Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
//Clean up previous data.
Shell(@`Qizmt del data_Input" + guid + @"*`);
Shell(@`Qizmt del data_Output" + guid + @"*`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
try
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `drop table " + tablename + @"`;
cmd.ExecuteNonQuery();
cmd.CommandText = `drop rindex " + indexname + @"`;
cmd.ExecuteNonQuery();
conn.Close();
}
catch
{
}
}
}
]]>
</Local>
</Job>
<Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
dfsoutput.WriteLine(`1`);
}
]]>
</Remote>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(5);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(5);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(10);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(10);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(14);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(21);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(25);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(80);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(80);
rkey.PutInt(2);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"4.bin@nInt,nInt</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(80);
rkey.PutInt(3);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(80);
rkey.PutInt(4);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(90);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(102);
rkey.PutInt(1);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>RDBMS_DBCORE</Using>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin data_Output" + guid + @"4.bin +data_Output" + guid + @".bin`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create table " + tablename + @" (id int, rid int)`;
cmd.ExecuteNonQuery();
cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`;
cmd.ExecuteNonQuery();
conn.Close();
}
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH ON id`;
cmd.ExecuteNonQuery();
conn.Close();
}
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>").Replace('`', '"');
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", "");
if (System.IO.Directory.Exists(tempdir))
{
System.IO.Directory.Delete(tempdir, true);
}
System.IO.Directory.CreateDirectory(tempdir);
string tempjobname = Guid.NewGuid().ToString();
System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job);
Exec.Shell("Qizmt importdir \"" + tempdir + "\"");
Exec.Shell("dspace exec " + tempjobname);
//Clean up
Exec.Shell(@"Qizmt del " + tempjobname);
System.IO.Directory.Delete(tempdir, true);
}
{
Console.WriteLine("RInsert/RDeleting rindex...");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText =
"rdelete from " + indexname + " where key = 80 and rid=1\0" +
"rdelete from " + indexname + " where key = 80 and rid=2\0" +
"rdelete from " + indexname + " where key = 80 and rid=3\0" +
"rdelete from " + indexname + " where key = 80 and rid=4\0" +
"rinsert into " + indexname + " values(10, 3) where key = 10\0" +
"rinsert into " + indexname + " values(10, 19) where key = 10\0" +
"rdelete from " + indexname + " where key = 10 and rid=19\0" +
"rinsert into " + indexname + " values(20, 1) where key = 20\0" +
"rinsert into " + indexname + " values(30, 1) where key = 30\0" +
"rinsert into " + indexname + " values(30, 2) where key = 30\0" +
"rinsert into " + indexname + " values(9, 1) where key = 9\0" +
"rinsert into " + indexname + " values(9, 2) where key = 9\0" +
"rinsert into " + indexname + " values(99, 1) where key = 99\0" +
"rdelete from " + indexname + " where key = 99 and rid=4444444\0" + //deleting existing key but secondary column value doesn't match
"rdelete from " + indexname + " where key = 100000 and rid=4444444\0" + //deleting non-existing key.
"rdelete from " + indexname + " where key = 5 and rid=2\0" +
"rinsert into " + indexname + " values(2, 1) where key = 2\0" +
"rinsert into " + indexname + " values(2, 19) where key = 2\0" +
"rinsert into " + indexname + " values(2, 2) where key = 2\0" +
"rdelete from " + indexname + " where key = 2 and rid=19\0";
cmd.ExecuteNonQuery();
conn.Close();
}
Dictionary <string, string> renamed = new Dictionary <string, string>();
{
Console.WriteLine("Invalidating chunks...");
string bulkget = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", "");
Exec.Shell("Qizmt bulkget " + bulkget + " RDBMS_Table_" + tablename);
string[] lines = System.IO.File.ReadAllLines(bulkget);
System.IO.File.Delete(bulkget);
foreach (string line in lines)
{
string[] parts = line.Split(' ');
string firsthost = parts[0].Split(';')[0];
string chunkname = parts[1];
string oldpath = @"\\" + firsthost + @"\" + qizmtrootdir + @"\" + chunkname;
string newpath = oldpath + "_regtest";
System.IO.File.Move(oldpath, newpath);
renamed.Add(oldpath, newpath);
}
}
{
Console.WriteLine("RSelecting from rindex after invalidating chunks, should fail over...");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "rselect * from " + indexname + " where key=80 or key = 2 or key = 5 or key=9 or key=10 or key=14 or key=20 or key=21 or key=25 or key=30 or key=90 or key=99 or key=102";
DbDataReader reader = cmd.ExecuteReader();
int cnt = 0;
List <KeyValuePair <int, int> > results = new List <KeyValuePair <int, int> >();
while (reader.Read())
{
int x = reader.GetInt32(0);
int y = reader.GetInt32(1);
KeyValuePair <int, int> row = new KeyValuePair <int, int>(x, y);
results.Add(row);
}
reader.Close();
conn.Close();
results.Sort(delegate(KeyValuePair <int, int> x, KeyValuePair <int, int> y)
{
if (x.Key != y.Key)
{
return(x.Key.CompareTo(y.Key));
}
else
{
return(x.Value.CompareTo(y.Value));
}
});
List <KeyValuePair <int, int> > expected = new List <KeyValuePair <int, int> >();
expected.Add(new KeyValuePair <int, int>(2, 1));
expected.Add(new KeyValuePair <int, int>(2, 2));
expected.Add(new KeyValuePair <int, int>(5, 1));
expected.Add(new KeyValuePair <int, int>(9, 1));
expected.Add(new KeyValuePair <int, int>(9, 2));
expected.Add(new KeyValuePair <int, int>(10, 1));
expected.Add(new KeyValuePair <int, int>(10, 2));
expected.Add(new KeyValuePair <int, int>(10, 3));
expected.Add(new KeyValuePair <int, int>(14, 1));
expected.Add(new KeyValuePair <int, int>(20, 1));
expected.Add(new KeyValuePair <int, int>(21, 1));
expected.Add(new KeyValuePair <int, int>(25, 1));
expected.Add(new KeyValuePair <int, int>(30, 1));
expected.Add(new KeyValuePair <int, int>(30, 2));
expected.Add(new KeyValuePair <int, int>(90, 1));
expected.Add(new KeyValuePair <int, int>(99, 1));
expected.Add(new KeyValuePair <int, int>(102, 1));
if (results.Count != expected.Count)
{
throw new Exception("Expected " + expected.Count.ToString() + " rows to be returned. But received " + results.Count.ToString() + " rows instead.");
}
for (int i = 0; i < results.Count; i++)
{
if (results[i].Key != expected[i].Key || results[i].Value != expected[i].Value)
{
throw new Exception("Row returned is different from expected results. Received row:(" + results[i].Key.ToString() + "," + results[i].Value.ToString() + "). Expected row:(" + expected[i].Key.ToString() + "," + expected[i].Value.ToString() + ")");
}
}
}
{
Console.WriteLine("Fixing chunks...");
foreach (KeyValuePair <string, string> pair in renamed)
{
System.IO.File.Move(pair.Value, pair.Key);
}
}
{
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = localhost";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "drop table " + tablename;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop rindex " + indexname;
cmd.ExecuteNonQuery();
conn.Close();
}
}
public static void RIndexSampling()
{
string guid = Guid.NewGuid().ToString().Replace("-", "");
string tablename = "rselect_test_" + guid;
string indexname = guid + "apple";
string indexname_order = guid + "order";
string[] allhosts = null;
{
string[] installs = Exec.Shell("Qizmt slaveinstalls").Trim()
.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
allhosts = new string[installs.Length];
for (int ip = 0; ip < installs.Length; ip++)
{
string[] parts = installs[ip].Split(' ');
allhosts[ip] = parts[0];
}
}
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider");
{
Console.WriteLine("Creating data, table, and index...");
string job = (@"<SourceCode>
<Jobs>
<Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
//Clean up previous data.
Shell(@`Qizmt del data_Input" + guid + @"*`);
Shell(@`Qizmt del data_Output" + guid + @"*`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
try
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `drop table " + tablename + @"`;
cmd.ExecuteNonQuery();
cmd.CommandText = `drop rindex " + indexname + @"`;
cmd.ExecuteNonQuery();
conn.Close();
}
catch
{
}
}
}
]]>
</Local>
</Job>
<Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
dfsoutput.WriteLine(`1`);
}
]]>
</Remote>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nChar(10)</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`101`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`102`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`103`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`104`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`105`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`106`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`107`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`108`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(1);
rkey.PutString(`109`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`201`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`202`, 10);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nChar(10)</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`203`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`204`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`205`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`206`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`207`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`208`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`209`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`210`, 10);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nChar(10)</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`211`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(2);
rkey.PutString(`212`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(3);
rkey.PutString(`301`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(3);
rkey.PutString(`302`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(3);
rkey.PutString(`303`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(3);
rkey.PutString(`304`, 10);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`data` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>nInt</KeyLength>
<DFSInput>dfs://data_Input" + guid + @".txt</DFSInput>
<DFSOutput>dfs://data_Output" + guid + @"4.bin@nInt,nChar(10)</DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Using>RDBMS_DBCORE</Using>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
output.Add(rkey.ToByteSlice(), ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(3);
rkey.PutString(`305`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
rkey.PutString(`401`, 10);
output.Add(rkey.ToByteSlice());
}
{
DbRecordset rkey = DbRecordset.Prepare();
rkey.PutInt(4);
rkey.PutString(`402`, 10);
output.Add(rkey.ToByteSlice());
}
}
]]>
</Reduce>
</MapReduce>
</Job>
<Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/>
<Add Reference=`System.Data.dll` Type=`system`/>
<Using>RDBMS_DBCORE</Using>
<Using>System.Data</Using>
<Using>System.Data.Common</Using>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin data_Output" + guid + @"4.bin +data_Output" + guid + @".bin`);
System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`);
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create table " + tablename + @" (id int, rid char(10))`;
cmd.ExecuteNonQuery();
cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`;
cmd.ExecuteNonQuery();
conn.Close();
}
using (DbConnection conn = fact.CreateConnection())
{
conn.ConnectionString = `Data Source = localhost`;
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH ON id `;
cmd.ExecuteNonQuery();
conn.Close();
}
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>").Replace('`', '"');
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", "");
if (System.IO.Directory.Exists(tempdir))
{
System.IO.Directory.Delete(tempdir, true);
}
System.IO.Directory.CreateDirectory(tempdir);
string tempjobname = Guid.NewGuid().ToString();
System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job);
Exec.Shell("Qizmt importdir \"" + tempdir + "\"");
Exec.Shell("dspace exec " + tempjobname);
//Clean up
Exec.Shell(@"Qizmt del " + tempjobname);
System.IO.Directory.Delete(tempdir, true);
}
{
Console.WriteLine("RSelecting samples from rindex...");
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled";
conn.Open();
DbCommand cmd = conn.CreateCommand();
//Getting something not in the table also.
cmd.CommandText = "rselect * from " + indexname + " sample 5 where key=-888 or key = 1 or key = 2 or key=3 or key=4 or key=-9999";
DbDataReader reader = cmd.ExecuteReader();
Dictionary <int, List <string> > results = new Dictionary <int, List <string> >();
while (reader.Read())
{
int x = reader.GetInt32(0);
string y = reader.GetString(1);
if (!results.ContainsKey(x))
{
results.Add(x, new List <string>());
}
results[x].Add(y);
}
reader.Close();
conn.Close();
Dictionary <int, Dictionary <string, int> > expected = new Dictionary <int, Dictionary <string, int> >();
expected.Add(1, new Dictionary <string, int>());
for (int i = 101; i <= 109; i++)
{
expected[1].Add(i.ToString(), 0);
}
expected.Add(2, new Dictionary <string, int>());
for (int i = 201; i <= 212; i++)
{
expected[2].Add(i.ToString(), 0);
}
expected.Add(3, new Dictionary <string, int>());
for (int i = 301; i <= 305; i++)
{
expected[3].Add(i.ToString(), 0);
}
expected.Add(4, new Dictionary <string, int>());
for (int i = 401; i <= 402; i++)
{
expected[4].Add(i.ToString(), 0);
}
//Make sure the correct number of samples are returned.
Dictionary <int, int> expectedCount = new Dictionary <int, int>();
expectedCount.Add(1, 5);
expectedCount.Add(2, 5);
expectedCount.Add(3, 5);
expectedCount.Add(4, 2);
foreach (int key in results.Keys)
{
if (!expected.ContainsKey(key))
{
throw new Exception("Unexpected key received: " + key.ToString());
}
foreach (string val in results[key])
{
if (!expected[key].ContainsKey(val))
{
throw new Exception("The pair: (" + key.ToString() + ", '" + val + "') is not found.");
}
if (expected[key][val] != 0)
{
throw new Exception("The pair: (" + key.ToString() + ", '" + val + "') is returned more than once.");
}
expected[key][val]++;
}
if (results[key].Count != expectedCount[key])
{
throw new Exception("Expected rows to be returned for key= " + key.ToString() + " is " + expectedCount[key].ToString() + ", but got " + results[key].Count.ToString() + " rows instead.");
}
}
}
{
DbConnection conn = fact.CreateConnection();
conn.ConnectionString = "Data Source = localhost";
conn.Open();
DbCommand cmd = conn.CreateCommand();
cmd.CommandText = "drop table " + tablename;
cmd.ExecuteNonQuery();
cmd.CommandText = "drop rindex " + indexname;
cmd.ExecuteNonQuery();
conn.Close();
}
}