public static void RSelectSpillageTest2() { string guid = Guid.NewGuid().ToString().Replace("-", ""); string tablename = "rselect_test_" + guid; string indexname = guid + "apple"; string indexname_order = guid + "order"; System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider"); { Console.WriteLine("Creating data, table, and index..."); string job = (@"<SourceCode> <Jobs> <Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`System.Data.dll` Type=`system`/> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { //Clean up previous data. Shell(@`Qizmt del data_Input" + guid + @"*`); Shell(@`Qizmt del data_Output" + guid + @"*`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { try { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `drop table " + tablename + @"`; cmd.ExecuteNonQuery(); cmd.CommandText = `drop rindex " + indexname + @"`; cmd.ExecuteNonQuery(); conn.Close(); } catch { } } } ]]> </Local> </Job> <Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`> <IOSettings> <JobType>remote</JobType> <DFS_IO> <DFSReader></DFSReader> <DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter> </DFS_IO> </IOSettings> <Remote> <![CDATA[ public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput) { dfsoutput.WriteLine(`1`); } ]]> </Remote> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { for(int i = 0; i < 2; i++) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutInt(0); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { for(int i = 0; i < 2; i++) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutInt(0); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { for(int i = 0; i < 2; i++) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutInt(i); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"4.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { for(int i = 0; i < 2; i++) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutInt(i); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"5.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { for(int i = 0; i < 2; i++) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); rkey.PutInt(i); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Add Reference=`System.Data.dll` Type=`system`/> <Using>RDBMS_DBCORE</Using> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin data_Output" + guid + @"4.bin data_Output" + guid + @"5.bin +data_Output" + guid + @".bin`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create table " + tablename + @" (id int, rid int)`; cmd.ExecuteNonQuery(); cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`; cmd.ExecuteNonQuery(); conn.Close(); } using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH ON id`; cmd.ExecuteNonQuery(); cmd.CommandText = `create rindex " + indexname_order + @" from " + tablename + @" pinmemoryHASH keepvalueorder ON id`; cmd.ExecuteNonQuery(); conn.Close(); } } ]]> </Local> </Job> </Jobs> </SourceCode>").Replace('`', '"'); string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", ""); if (System.IO.Directory.Exists(tempdir)) { System.IO.Directory.Delete(tempdir, true); } System.IO.Directory.CreateDirectory(tempdir); string tempjobname = Guid.NewGuid().ToString(); System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job); Exec.Shell("Qizmt importdir \"" + tempdir + "\""); Exec.Shell("dspace exec " + tempjobname); //Clean up Exec.Shell(@"Qizmt del " + tempjobname); System.IO.Directory.Delete(tempdir, true); } { Console.WriteLine("RSelecting from rindex..."); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = localhost; rindex=pooled"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "rselect * from " + indexname + " where key = 2"; DbDataReader reader = cmd.ExecuteReader(); int cnt = 0; while (reader.Read()) { if (reader.GetInt32(0) != 2) { throw new Exception("Expected value."); } cnt++; } reader.Close(); if (cnt != 6) { throw new Exception("Expected 6 rows, but received only " + cnt.ToString()); } conn.Close(); } { DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = localhost"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "drop table " + tablename; cmd.ExecuteNonQuery(); cmd.CommandText = "drop rindex " + indexname; cmd.ExecuteNonQuery(); cmd.CommandText = "drop rindex " + indexname_order; cmd.ExecuteNonQuery(); conn.Close(); } }
public static void RIndexOutlierDelete() { string guid = Guid.NewGuid().ToString().Replace("-", ""); string tablename = "rselect_test_" + guid; string indexname = guid + "apple"; string indexname_order = guid + "order"; string[] allhosts = null; { string[] installs = Exec.Shell("Qizmt slaveinstalls").Trim() .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); allhosts = new string[installs.Length]; for (int ip = 0; ip < installs.Length; ip++) { string[] parts = installs[ip].Split(' '); allhosts[ip] = parts[0]; } } System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider"); { Console.WriteLine("Creating data, table, and index..."); string job = (@"<SourceCode> <Jobs> <Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`System.Data.dll` Type=`system`/> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { //Clean up previous data. Shell(@`Qizmt del data_Input" + guid + @"*`); Shell(@`Qizmt del data_Output" + guid + @"*`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { try { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `drop table " + tablename + @"`; cmd.ExecuteNonQuery(); cmd.CommandText = `drop rindex " + indexname + @"`; cmd.ExecuteNonQuery(); conn.Close(); } catch { } } } ]]> </Local> </Job> <Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`> <IOSettings> <JobType>remote</JobType> <DFS_IO> <DFSReader></DFSReader> <DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter> </DFS_IO> </IOSettings> <Remote> <![CDATA[ public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput) { dfsoutput.WriteLine(`1`); } ]]> </Remote> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(-2); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(-2); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(-2); rkey.PutInt(3); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(5); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(5); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(5); rkey.PutInt(3); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(5); rkey.PutInt(4); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(10); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(10); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(10); rkey.PutInt(3); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(19); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(19); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(20); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(20); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(20); rkey.PutInt(3); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(20); rkey.PutInt(4); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(20); rkey.PutInt(5); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(20); rkey.PutInt(6); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(21); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(21); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(3); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(4); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(5); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(6); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(30); rkey.PutInt(7); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Add Reference=`System.Data.dll` Type=`system`/> <Using>RDBMS_DBCORE</Using> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin +data_Output" + guid + @".bin`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create table " + tablename + @" (id int, rid int)`; cmd.ExecuteNonQuery(); cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`; cmd.ExecuteNonQuery(); conn.Close(); } using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH OUTLIER Delete 3 ON id`; cmd.ExecuteNonQuery(); conn.Close(); } } ]]> </Local> </Job> </Jobs> </SourceCode>").Replace('`', '"'); string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", ""); if (System.IO.Directory.Exists(tempdir)) { System.IO.Directory.Delete(tempdir, true); } System.IO.Directory.CreateDirectory(tempdir); string tempjobname = Guid.NewGuid().ToString(); System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job); Exec.Shell("Qizmt importdir \"" + tempdir + "\""); Exec.Shell("dspace exec " + tempjobname); //Clean up Exec.Shell(@"Qizmt del " + tempjobname); System.IO.Directory.Delete(tempdir, true); } { Console.WriteLine("RInsert/RDeleting rindex..."); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "rinsert into " + indexname + " values(22,1) where key = 22\0" + "rinsert into " + indexname + " values(22,2) where key = 22\0" + "rinsert into " + indexname + " values(22,3) where key = 22\0" + "rinsert into " + indexname + " values(22,4) where key = 22\0" + "rinsert into " + indexname + " values(21,3) where key = 21\0" + "rinsert into " + indexname + " values(21,4) where key = 21\0" + "rinsert into " + indexname + " values(21,5) where key = 21\0" + "rdelete from " + indexname + " where key = 21 and rid = 4\0" + "rdelete from " + indexname + " where key = 21 and rid = 5\0" + "rinsert into " + indexname + " values(8,1) where key = 8\0" + "rinsert into " + indexname + " values(8,2) where key = 8\0" + "rinsert into " + indexname + " values(8,3) where key = 8\0" + "rinsert into " + indexname + " values(8,4) where key = 8\0" + "rinsert into " + indexname + " values(6,1) where key = 6\0" + "rinsert into " + indexname + " values(6,2) where key = 6\0" + "rinsert into " + indexname + " values(5,5) where key = 5\0" + "rinsert into " + indexname + " values(5,6) where key = 5\0" + "rinsert into " + indexname + " values(-2,4) where key = -2\0"; cmd.ExecuteNonQuery(); conn.Close(); } { Console.WriteLine("RSelecting from rindex..."); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "rselect * from " + indexname + " where key = -2 or key = 5 or key = 6 or key = 8 or key= 10 or key = 19 or key=20 or key = 21 or key = 22 or key=30"; DbDataReader reader = cmd.ExecuteReader(); int cnt = 0; List <KeyValuePair <int, int> > results = new List <KeyValuePair <int, int> >(); while (reader.Read()) { int x = reader.GetInt32(0); int y = reader.GetInt32(1); KeyValuePair <int, int> row = new KeyValuePair <int, int>(x, y); results.Add(row); } reader.Close(); conn.Close(); results.Sort(delegate(KeyValuePair <int, int> x, KeyValuePair <int, int> y) { if (x.Key != y.Key) { return(x.Key.CompareTo(y.Key)); } else { return(x.Value.CompareTo(y.Value)); } }); List <KeyValuePair <int, int> > expected = new List <KeyValuePair <int, int> >(); expected.Add(new KeyValuePair <int, int>(5, 1)); expected.Add(new KeyValuePair <int, int>(5, 2)); expected.Add(new KeyValuePair <int, int>(6, 1)); expected.Add(new KeyValuePair <int, int>(6, 2)); expected.Add(new KeyValuePair <int, int>(10, 1)); expected.Add(new KeyValuePair <int, int>(10, 2)); expected.Add(new KeyValuePair <int, int>(10, 3)); expected.Add(new KeyValuePair <int, int>(19, 1)); expected.Add(new KeyValuePair <int, int>(19, 2)); expected.Add(new KeyValuePair <int, int>(21, 1)); expected.Add(new KeyValuePair <int, int>(21, 2)); expected.Add(new KeyValuePair <int, int>(21, 3)); if (results.Count != expected.Count) { throw new Exception("Expected " + expected.Count.ToString() + " rows to be returned. But received " + results.Count.ToString() + " rows instead."); } for (int i = 0; i < results.Count; i++) { if (results[i].Key != expected[i].Key || results[i].Value != expected[i].Value) { throw new Exception("Row returned is different from expected results. Received row:(" + results[i].Key.ToString() + "," + results[i].Value.ToString() + "). Expected row:(" + expected[i].Key.ToString() + "," + expected[i].Value.ToString() + ")"); } } } { DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = localhost"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "drop table " + tablename; cmd.ExecuteNonQuery(); cmd.CommandText = "drop rindex " + indexname; cmd.ExecuteNonQuery(); conn.Close(); } }
public static void FaultTolerantExecution() { if (!MySpace.DataMining.AELight.FTTest.enabled) { throw new Exception("TESTFAULTTOLERANT is not #defined. Need Qizmt build with all #define TESTFAULTTOLERANT uncommented."); } string guid = "A275169D14B34df48229FC3F43A0AA31"; string tblUsers = "users_" + guid; string tblPageviews = "pageviews_" + guid; string tblPageToNames = "pagetonames_" + guid; string controlfile = @"\\" + MySpace.DataMining.AELight.Surrogate.MasterHost + @"\c$\temp\" + MySpace.DataMining.AELight.FTTest.controlfilename; string tempdir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + @"\" + Guid.NewGuid().ToString(); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("Qizmt_DataProvider"); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = localhost; mr bypass=10; fault tolerant execution=enabled"; DbCommand cmd = conn.CreateCommand(); try { { System.IO.Directory.CreateDirectory(tempdir); #region genPageViews System.IO.File.WriteAllText(tempdir + @"\genPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml", @"<SourceCode> <Jobs> <Job Name=`Preprocessing` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Local> <![CDATA[ public virtual void Local() { Shell(@`Qizmt del tblPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.bin`); } ]]> </Local> </Job> <Job Name=`CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`> <IOSettings> <JobType>remote</JobType> <DFS_IO> <DFSReader></DFSReader> <DFSWriter>dfs://tblPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.bin@8</DFSWriter> </DFS_IO> </IOSettings> <Remote> <![CDATA[ public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput) { List<byte> buf = new List<byte>(24); { recordset rs = recordset.Prepare(); rs.PutInt(100); rs.PutInt(1); rs.ToByteSlice().AppendTo(buf); dfsoutput.WriteRecord(buf); } { buf.Clear(); recordset rs = recordset.Prepare(); rs.PutInt(101); rs.PutInt(1); rs.ToByteSlice().AppendTo(buf); dfsoutput.WriteRecord(buf); } { buf.Clear(); recordset rs = recordset.Prepare(); rs.PutInt(300); rs.PutInt(3); rs.ToByteSlice().AppendTo(buf); dfsoutput.WriteRecord(buf); } { buf.Clear(); recordset rs = recordset.Prepare(); rs.PutInt(301); rs.PutInt(3); rs.ToByteSlice().AppendTo(buf); dfsoutput.WriteRecord(buf); } { buf.Clear(); recordset rs = recordset.Prepare(); rs.PutInt(302); rs.PutInt(3); rs.ToByteSlice().AppendTo(buf); dfsoutput.WriteRecord(buf); } { buf.Clear(); recordset rs = recordset.Prepare(); rs.PutInt(400); rs.PutInt(4); rs.ToByteSlice().AppendTo(buf); dfsoutput.WriteRecord(buf); } } ]]> </Remote> </Job> </Jobs> </SourceCode> ".Replace('`', '"')); #endregion #region genPageToNames System.IO.File.WriteAllText(tempdir + @"\genPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml", @"<SourceCode> <Jobs> <Job Name=`Preprocessing` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Local> <![CDATA[ public virtual void Local() { Shell(@`Qizmt del tblPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.txt`); } ]]> </Local> </Job> <Job Name=`CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`> <IOSettings> <JobType>remote</JobType> <DFS_IO> <DFSReader></DFSReader> <DFSWriter>dfs://tblPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.txt</DFSWriter> </DFS_IO> </IOSettings> <Remote> <![CDATA[ public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput) { //Create sample data. dfsoutput.WriteLine(`90,a`); dfsoutput.WriteLine(`91,b`); dfsoutput.WriteLine(`92,c`); dfsoutput.WriteLine(`93,d`); dfsoutput.WriteLine(`94,e`); } ]]> </Remote> </Job> </Jobs> </SourceCode> ".Replace('`', '"')); #endregion Exec.Shell("qizmt importdir \"" + tempdir + "\""); Exec.Shell("qizmt exec genPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml"); Exec.Shell("qizmt exec genPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2.xml"); } conn.Open(); #region createtables { Console.WriteLine("Creating tables..."); try { cmd.CommandText = "drop table " + tblUsers; cmd.ExecuteNonQuery(); cmd.CommandText = "drop table " + tblPageviews; cmd.ExecuteNonQuery(); cmd.CommandText = "drop table " + tblPageToNames; cmd.ExecuteNonQuery(); conn.Close(); //flush } catch { } conn.Open(); cmd.CommandText = "create table " + tblUsers + " (userid int, username char(50))"; cmd.ExecuteNonQuery(); cmd.CommandText = "insert into " + tblUsers + " values (1, 'john')"; cmd.ExecuteNonQuery(); cmd.CommandText = "insert into " + tblUsers + " values (2, 'mary')"; cmd.ExecuteNonQuery(); cmd.CommandText = "insert into " + tblUsers + " values (3, 'joe')"; cmd.ExecuteNonQuery(); cmd.CommandText = "create table " + tblPageviews + " (pageid int, uid int)"; cmd.ExecuteNonQuery(); cmd.CommandText = "create table " + tblPageToNames + " (pageid int, username char(50))"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); } #endregion #region testIMPORT { string phase = "exchangeowned"; Console.WriteLine("Testing IMPORT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "INSERT INTO " + tblPageviews + " IMPORT 'tblPageViews_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.bin'"; cmd.ExecuteNonQuery(); conn.Close(); conn.Open(); System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "select * from " + tblPageviews; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); int userid = reader.GetInt32(1); if (!(pageid == 100 && userid == 1) && !(pageid == 101 && userid == 1) && !(pageid == 300 && userid == 3) && !(pageid == 301 && userid == 3) && !(pageid == 302 && userid == 3) && !(pageid == 400 && userid == 4)) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 6) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testIMPORTLINES { string phase = "reduce"; Console.WriteLine("Testing IMPORTLINES in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "INSERT INTO " + tblPageToNames + " IMPORTLINES 'tblPageToNames_873200A2-EFD4-4136-9209-A807CF8BA3C2_Output.txt'"; cmd.ExecuteNonQuery(); conn.Close(); conn.Open(); System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "select * from " + tblPageToNames; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); string name = reader.GetString(1); if (!(pageid == 90 && name == "a") && !(pageid == 91 && name == "b") && !(pageid == 92 && name == "c") && !(pageid == 93 && name == "d") && !(pageid == 94 && name == "e")) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 5) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testSELECT { string phase = "map"; Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "select * from " + tblUsers + " where userid = 2"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; if (reader.GetString(1) != "mary") { throw new Exception("Unexpected value for field username"); } } reader.Close(); if (count != 1) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testSELECT { string phase = "exchangeremote"; Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "select * from " + tblUsers + " where userid > 0"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; string username = reader.GetString(1); if (username != "mary" && username != "john" && username != "joe") { throw new Exception("Unexpected value for field username"); } } reader.Close(); if (count != 3) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testSELECT { string phase = "exchangeowned"; Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "select username from " + tblUsers + " where userid=3"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; if (reader.GetString(0) != "joe") { throw new Exception("Unexpected value for field username"); } } reader.Close(); if (count != 1) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testSELECT { string phase = "sort"; Console.WriteLine("Testing SELECT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "select username from " + tblUsers + " where abs(userid)=3"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; if (reader.GetString(0) != "joe") { throw new Exception("Unexpected value for field username"); } } reader.Close(); if (count != 1) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testSELECT { Console.WriteLine("Testing SELECT in FTE mode with no failure..."); System.IO.File.WriteAllText(controlfile, ""); //Do not simulate failure since FTE is not enabled for SELECT func(column) cmd.CommandText = "select max(pageid), uid from " + tblPageviews + " group by uid"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); int uid = reader.GetInt32(1); if (!(pageid == 101 && uid == 1) && !(pageid == 302 && uid == 3) && !(pageid == 400 && uid == 4)) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 3) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testINNERJOIN { string phase = "reduce"; Console.WriteLine("Testing INNER JOIN in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "SELECT * FROM " + tblPageviews + " inner join " + tblUsers + " on uid=userid"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); string username = reader.GetString(3); if (!(pageid == 100 && username == "john") && !(pageid == 101 && username == "john") && !(pageid == 300 && username == "joe") && !(pageid == 301 && username == "joe") && !(pageid == 302 && username == "joe")) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 5) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testINNERJOIN { string phase = "sort"; Console.WriteLine("Testing INNER JOIN in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } long jid = GetJobID("RDBMS_JoinOn.DBCORE"); if (jid != -1) { throw new Exception("Expected jobid = -1"); } List <string> errors = new List <string>(); System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { while (jid == -1) { Console.Write("."); System.Threading.Thread.Sleep(1000); jid = GetJobID("RDBMS_JoinOn.DBCORE"); } Console.WriteLine("jid={0}", jid); for (; ;) { Console.Write("."); System.Threading.Thread.Sleep(1000); if (GetJobOutputIndexOf(jid, "Replicating") > -1) { //Can clear control file for the next job run. ClearControlFile(controlfile); Console.WriteLine("control file cleared"); break; } } })); tm.IsBackground = true; tm.Start(); System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { cmd.CommandText = "select max(pageid), username from " + tblPageviews + " inner join " + tblUsers + " on uid = userid group by username"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); string username = reader.GetString(1); if (!(pageid == 101 && username == "john") && !(pageid == 302 && username == "joe")) { errors.Add("Unexpected value"); throw new Exception("Unexpected value"); } } reader.Close(); if (count != 2) { errors.Add("Unexpected rows returned"); throw new Exception("Unexpected rows returned"); } })); ts.IsBackground = true; ts.Start(); tm.Join(); ts.Join(); if (errors.Count > 0) { StringBuilder sb = new StringBuilder(); foreach (string err in errors) { sb.Append(err); sb.Append(Environment.NewLine); } throw new Exception("Error during job run:" + sb.ToString()); } Console.WriteLine("Done"); } #endregion #region testLEFTJOIN { string phase = "replication"; Console.WriteLine("Testing LEFT OUTER JOIN in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "SELECT * FROM " + tblPageviews + " left outer join " + tblUsers + " on uid=userid"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = (int)reader[0]; string username = DBNull.Value.Equals(reader[3]) ? null : (string)reader[3]; if (!(pageid == 100 && username == "john") && !(pageid == 101 && username == "john") && !(pageid == 300 && username == "joe") && !(pageid == 301 && username == "joe") && !(pageid == 302 && username == "joe") && !(pageid == 400 && username == null)) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 6) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testRIGHTJOIN { string phase = "map"; Console.WriteLine("Testing RIGHT OUTER JOIN in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "SELECT * FROM " + tblPageviews + " right outer join " + tblUsers + " on uid=userid"; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int userid = (int)reader[2]; int pageid = DBNull.Value.Equals(reader[0]) ? -1 : (int)reader[0]; if (!(userid == 1 && pageid == 100) && !(userid == 1 && pageid == 101) && !(userid == 3 && pageid == 300) && !(userid == 3 && pageid == 301) && !(userid == 3 && pageid == 302) && !(userid == 2 && pageid == -1)) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 6) { throw new Exception("Unexpected rows returned"); } Console.WriteLine("Done"); } #endregion #region testDELETE { { string phase = "exchangeremote"; Console.WriteLine("Testing DELETE in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "DELETE FROM " + tblPageToNames + " where username='******'"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); Console.WriteLine("+"); } { string phase = "exchangeowned"; Console.WriteLine("Testing DELETE in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "DELETE FROM " + tblPageToNames + " where abs(pageid)= 93"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); Console.WriteLine("+"); } { string phase = "sort"; Console.WriteLine("Testing DELETE in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "delete from " + tblPageToNames + " where mod(pageid, 2) = 0"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); Console.WriteLine("+"); } { System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "select * from " + tblPageToNames; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; } reader.Close(); if (count != 0) { throw new Exception("Unexpected number of rows returned"); } } Console.WriteLine("Done"); } #endregion #region testINSERT { { string phase = "reduce"; Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } cmd.CommandText = "insert into " + tblPageToNames + " select userid, username from " + tblUsers + " where userid > 2"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); Console.WriteLine("+"); } { Console.WriteLine("Testing INSERT in FTE mode with no failure"); System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "insert into " + tblPageToNames + " select abs(userid), username from " + tblUsers + " where userid = 2"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); Console.WriteLine("+"); } { Console.WriteLine("Testing INSERT in FTE mode with no failure"); cmd.CommandText = "insert into " + tblPageToNames + " select userid, username from " + tblUsers + " where userid = 1 order by userid"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); Console.WriteLine("+"); } { System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "select * from " + tblPageToNames; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); string username = reader.GetString(1); if (!(pageid == 1 && username == "john") && !(pageid == 2 && username == "mary") && !(pageid == 3 && username == "joe")) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 3) { throw new Exception("Unexpected number of rows returned"); } } Console.WriteLine("Done"); } #endregion #region testINSERT { //prepare { System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "delete from " + tblPageToNames + " where pageid > 0"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); } { string phase = "exchangeowned"; Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } long jid = GetJobID("RDBMS_JoinOn.DBCORE"); if (jid != -1) { throw new Exception("Expected jobid = -1"); } System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { while (jid == -1) { Console.Write("."); System.Threading.Thread.Sleep(1000); jid = GetJobID("RDBMS_JoinOn.DBCORE"); } Console.WriteLine("jid={0}", jid); for (; ;) { Console.Write("."); System.Threading.Thread.Sleep(1000); if (GetJobOutputIndexOf(jid, "Replicating") > -1) { //Can clear control file for the next job run. ClearControlFile(controlfile); Console.WriteLine("control file cleared"); break; } } })); tm.IsBackground = true; tm.Start(); System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { cmd.CommandText = "insert into " + tblPageToNames + " select abs(pageid), username from " + tblPageviews + " inner join " + tblUsers + " on uid = userid where abs(userid) = 1"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); })); ts.IsBackground = true; ts.Start(); tm.Join(); ts.Join(); Console.WriteLine("+"); } { string phase = "exchangeremote"; Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } long jid = GetJobID("RDBMS_JoinOn.DBCORE"); if (jid != -1) { throw new Exception("Expected jobid = -1"); } System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { while (jid == -1) { Console.Write("."); System.Threading.Thread.Sleep(1000); jid = GetJobID("RDBMS_JoinOn.DBCORE"); } Console.WriteLine("jid={0}", jid); for (; ;) { Console.Write("."); System.Threading.Thread.Sleep(1000); if (GetJobOutputIndexOf(jid, "Replicating") > -1) { //Can clear control file for the next job run. ClearControlFile(controlfile); Console.WriteLine("control file cleared"); break; } } })); tm.IsBackground = true; tm.Start(); System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { cmd.CommandText = "insert into " + tblPageToNames + " select abs(pageid), username from " + tblPageviews + " inner join " + tblUsers + " on uid = userid where abs(userid) = 3 order by pageid"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); })); ts.IsBackground = true; ts.Start(); tm.Join(); ts.Join(); Console.WriteLine("+"); } { string phase = "sort"; Console.WriteLine("Testing INSERT in FTE mode with failure at {0}...", phase); using (System.IO.StreamWriter w = new System.IO.StreamWriter(controlfile)) { w.WriteLine("{1}:" + phase); } long jid = GetJobID("RDBMS_JoinOn.DBCORE"); if (jid != -1) { throw new Exception("Expected jobid = -1"); } System.Threading.Thread tm = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { while (jid == -1) { Console.Write("."); System.Threading.Thread.Sleep(1000); jid = GetJobID("RDBMS_JoinOn.DBCORE"); } Console.WriteLine("jid={0}", jid); for (; ;) { Console.Write("."); System.Threading.Thread.Sleep(1000); if (GetJobOutputIndexOf(jid, "Replicating") > -1) { //Can clear control file for the next job run. ClearControlFile(controlfile); Console.WriteLine("control file cleared"); break; } } })); tm.IsBackground = true; tm.Start(); System.Threading.Thread ts = new System.Threading.Thread(new System.Threading.ThreadStart(delegate() { cmd.CommandText = "insert into " + tblPageToNames + " select max(userid), username from " + tblPageviews + " inner join " + tblUsers + " on uid = userid group by username"; cmd.ExecuteNonQuery(); conn.Close(); //flush conn.Open(); })); ts.IsBackground = true; ts.Start(); tm.Join(); ts.Join(); Console.WriteLine("+"); } { System.IO.File.WriteAllText(controlfile, ""); cmd.CommandText = "select * from " + tblPageToNames; DbDataReader reader = cmd.ExecuteReader(); int count = 0; while (reader.Read()) { count++; int pageid = reader.GetInt32(0); string username = reader.GetString(1); if (!(pageid == 100 && username == "john") && !(pageid == 101 && username == "john") && !(pageid == 300 && username == "joe") && !(pageid == 301 && username == "joe") && !(pageid == 302 && username == "joe") && !(pageid == 1 && username == "john") && !(pageid == 3 && username == "joe")) { throw new Exception("Unexpected value"); } } reader.Close(); if (count != 7) { throw new Exception("Unexpected number of rows returned"); } } Console.WriteLine("Done"); } #endregion #region cleanup try { cmd.CommandText = "drop table " + tblUsers; cmd.ExecuteNonQuery(); cmd.CommandText = "drop table " + tblPageviews; cmd.ExecuteNonQuery(); cmd.CommandText = "drop table " + tblPageToNames; cmd.ExecuteNonQuery(); } catch { } #endregion } finally { if (conn.State == ConnectionState.Open) { try { conn.Close(); } catch (Exception e) { Console.WriteLine("Closing error: " + e.ToString()); } } System.IO.Directory.Delete(tempdir, true); Exec.Shell("qizmt del *873200A2-EFD4-4136-9209-A807CF8BA3C2*"); System.IO.File.Delete(controlfile); } }
static int GetJobOutputIndexOf(long jobid, string token) { string output = Exec.Shell("qizmt viewjob " + jobid.ToString()); return(output.IndexOf(token, StringComparison.OrdinalIgnoreCase)); }
public static void RIndexReplication() { { int replicationFactor = 0; string[] lines = Exec.Shell("Qizmt replicationview").Split('\n'); for (int i = 0; i < lines.Length; i++) { string line = lines[i].Trim(); if (line.Length > 0) { int del = line.LastIndexOf(' '); replicationFactor = Int32.Parse(line.Substring(del + 1)); if (replicationFactor > 0) { break; } } } if (replicationFactor < 2) { throw new Exception("Cannot run RIndexReplication regression test when replication factor is less than 2."); } } string guid = Guid.NewGuid().ToString().Replace("-", ""); string tablename = "rselect_test_" + guid; string indexname = guid + "apple"; string indexname_order = guid + "order"; string[] allhosts = null; string qizmtrootdir = ""; { string[] installs = Exec.Shell("Qizmt slaveinstalls").Trim() .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); allhosts = new string[installs.Length]; for (int ip = 0; ip < installs.Length; ip++) { string[] parts = installs[ip].Split(' '); string installpath = parts[1]; int del = installpath.IndexOf(@"\", 2); qizmtrootdir = installpath.Substring(del + 1); allhosts[ip] = parts[0]; } } System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider"); { Console.WriteLine("Creating data, table, and index..."); string job = (@"<SourceCode> <Jobs> <Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`System.Data.dll` Type=`system`/> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { //Clean up previous data. Shell(@`Qizmt del data_Input" + guid + @"*`); Shell(@`Qizmt del data_Output" + guid + @"*`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { try { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `drop table " + tablename + @"`; cmd.ExecuteNonQuery(); cmd.CommandText = `drop rindex " + indexname + @"`; cmd.ExecuteNonQuery(); conn.Close(); } catch { } } } ]]> </Local> </Job> <Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`> <IOSettings> <JobType>remote</JobType> <DFS_IO> <DFSReader></DFSReader> <DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter> </DFS_IO> </IOSettings> <Remote> <![CDATA[ public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput) { dfsoutput.WriteLine(`1`); } ]]> </Remote> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(5); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(5); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(10); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(10); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(14); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(21); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(25); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(80); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(80); rkey.PutInt(2); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"4.bin@nInt,nInt</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(80); rkey.PutInt(3); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(80); rkey.PutInt(4); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(90); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(102); rkey.PutInt(1); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Add Reference=`System.Data.dll` Type=`system`/> <Using>RDBMS_DBCORE</Using> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin data_Output" + guid + @"4.bin +data_Output" + guid + @".bin`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create table " + tablename + @" (id int, rid int)`; cmd.ExecuteNonQuery(); cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`; cmd.ExecuteNonQuery(); conn.Close(); } using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH ON id`; cmd.ExecuteNonQuery(); conn.Close(); } } ]]> </Local> </Job> </Jobs> </SourceCode>").Replace('`', '"'); string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", ""); if (System.IO.Directory.Exists(tempdir)) { System.IO.Directory.Delete(tempdir, true); } System.IO.Directory.CreateDirectory(tempdir); string tempjobname = Guid.NewGuid().ToString(); System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job); Exec.Shell("Qizmt importdir \"" + tempdir + "\""); Exec.Shell("dspace exec " + tempjobname); //Clean up Exec.Shell(@"Qizmt del " + tempjobname); System.IO.Directory.Delete(tempdir, true); } { Console.WriteLine("RInsert/RDeleting rindex..."); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "rdelete from " + indexname + " where key = 80 and rid=1\0" + "rdelete from " + indexname + " where key = 80 and rid=2\0" + "rdelete from " + indexname + " where key = 80 and rid=3\0" + "rdelete from " + indexname + " where key = 80 and rid=4\0" + "rinsert into " + indexname + " values(10, 3) where key = 10\0" + "rinsert into " + indexname + " values(10, 19) where key = 10\0" + "rdelete from " + indexname + " where key = 10 and rid=19\0" + "rinsert into " + indexname + " values(20, 1) where key = 20\0" + "rinsert into " + indexname + " values(30, 1) where key = 30\0" + "rinsert into " + indexname + " values(30, 2) where key = 30\0" + "rinsert into " + indexname + " values(9, 1) where key = 9\0" + "rinsert into " + indexname + " values(9, 2) where key = 9\0" + "rinsert into " + indexname + " values(99, 1) where key = 99\0" + "rdelete from " + indexname + " where key = 99 and rid=4444444\0" + //deleting existing key but secondary column value doesn't match "rdelete from " + indexname + " where key = 100000 and rid=4444444\0" + //deleting non-existing key. "rdelete from " + indexname + " where key = 5 and rid=2\0" + "rinsert into " + indexname + " values(2, 1) where key = 2\0" + "rinsert into " + indexname + " values(2, 19) where key = 2\0" + "rinsert into " + indexname + " values(2, 2) where key = 2\0" + "rdelete from " + indexname + " where key = 2 and rid=19\0"; cmd.ExecuteNonQuery(); conn.Close(); } Dictionary <string, string> renamed = new Dictionary <string, string>(); { Console.WriteLine("Invalidating chunks..."); string bulkget = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", ""); Exec.Shell("Qizmt bulkget " + bulkget + " RDBMS_Table_" + tablename); string[] lines = System.IO.File.ReadAllLines(bulkget); System.IO.File.Delete(bulkget); foreach (string line in lines) { string[] parts = line.Split(' '); string firsthost = parts[0].Split(';')[0]; string chunkname = parts[1]; string oldpath = @"\\" + firsthost + @"\" + qizmtrootdir + @"\" + chunkname; string newpath = oldpath + "_regtest"; System.IO.File.Move(oldpath, newpath); renamed.Add(oldpath, newpath); } } { Console.WriteLine("RSelecting from rindex after invalidating chunks, should fail over..."); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "rselect * from " + indexname + " where key=80 or key = 2 or key = 5 or key=9 or key=10 or key=14 or key=20 or key=21 or key=25 or key=30 or key=90 or key=99 or key=102"; DbDataReader reader = cmd.ExecuteReader(); int cnt = 0; List <KeyValuePair <int, int> > results = new List <KeyValuePair <int, int> >(); while (reader.Read()) { int x = reader.GetInt32(0); int y = reader.GetInt32(1); KeyValuePair <int, int> row = new KeyValuePair <int, int>(x, y); results.Add(row); } reader.Close(); conn.Close(); results.Sort(delegate(KeyValuePair <int, int> x, KeyValuePair <int, int> y) { if (x.Key != y.Key) { return(x.Key.CompareTo(y.Key)); } else { return(x.Value.CompareTo(y.Value)); } }); List <KeyValuePair <int, int> > expected = new List <KeyValuePair <int, int> >(); expected.Add(new KeyValuePair <int, int>(2, 1)); expected.Add(new KeyValuePair <int, int>(2, 2)); expected.Add(new KeyValuePair <int, int>(5, 1)); expected.Add(new KeyValuePair <int, int>(9, 1)); expected.Add(new KeyValuePair <int, int>(9, 2)); expected.Add(new KeyValuePair <int, int>(10, 1)); expected.Add(new KeyValuePair <int, int>(10, 2)); expected.Add(new KeyValuePair <int, int>(10, 3)); expected.Add(new KeyValuePair <int, int>(14, 1)); expected.Add(new KeyValuePair <int, int>(20, 1)); expected.Add(new KeyValuePair <int, int>(21, 1)); expected.Add(new KeyValuePair <int, int>(25, 1)); expected.Add(new KeyValuePair <int, int>(30, 1)); expected.Add(new KeyValuePair <int, int>(30, 2)); expected.Add(new KeyValuePair <int, int>(90, 1)); expected.Add(new KeyValuePair <int, int>(99, 1)); expected.Add(new KeyValuePair <int, int>(102, 1)); if (results.Count != expected.Count) { throw new Exception("Expected " + expected.Count.ToString() + " rows to be returned. But received " + results.Count.ToString() + " rows instead."); } for (int i = 0; i < results.Count; i++) { if (results[i].Key != expected[i].Key || results[i].Value != expected[i].Value) { throw new Exception("Row returned is different from expected results. Received row:(" + results[i].Key.ToString() + "," + results[i].Value.ToString() + "). Expected row:(" + expected[i].Key.ToString() + "," + expected[i].Value.ToString() + ")"); } } } { Console.WriteLine("Fixing chunks..."); foreach (KeyValuePair <string, string> pair in renamed) { System.IO.File.Move(pair.Value, pair.Key); } } { DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = localhost"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "drop table " + tablename; cmd.ExecuteNonQuery(); cmd.CommandText = "drop rindex " + indexname; cmd.ExecuteNonQuery(); conn.Close(); } }
public static void RIndexSampling() { string guid = Guid.NewGuid().ToString().Replace("-", ""); string tablename = "rselect_test_" + guid; string indexname = guid + "apple"; string indexname_order = guid + "order"; string[] allhosts = null; { string[] installs = Exec.Shell("Qizmt slaveinstalls").Trim() .Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); allhosts = new string[installs.Length]; for (int ip = 0; ip < installs.Length; ip++) { string[] parts = installs[ip].Split(' '); allhosts[ip] = parts[0]; } } System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory("DSpace_DataProvider"); { Console.WriteLine("Creating data, table, and index..."); string job = (@"<SourceCode> <Jobs> <Job Name=`Cleanup_Previous_Data` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`System.Data.dll` Type=`system`/> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { //Clean up previous data. Shell(@`Qizmt del data_Input" + guid + @"*`); Shell(@`Qizmt del data_Output" + guid + @"*`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { try { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `drop table " + tablename + @"`; cmd.ExecuteNonQuery(); cmd.CommandText = `drop rindex " + indexname + @"`; cmd.ExecuteNonQuery(); conn.Close(); } catch { } } } ]]> </Local> </Job> <Job Name=`data_CreateSampleData` Custodian=`` Email=`` Description=`Create sample data`> <IOSettings> <JobType>remote</JobType> <DFS_IO> <DFSReader></DFSReader> <DFSWriter>dfs://data_Input" + guid + @".txt</DFSWriter> </DFS_IO> </IOSettings> <Remote> <![CDATA[ public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput) { dfsoutput.WriteLine(`1`); } ]]> </Remote> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"1.bin@nInt,nChar(10)</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`101`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`102`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`103`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`104`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`105`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`106`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`107`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`108`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(1); rkey.PutString(`109`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`201`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`202`, 10); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"2.bin@nInt,nChar(10)</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`203`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`204`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`205`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`206`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`207`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`208`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`209`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`210`, 10); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"3.bin@nInt,nChar(10)</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`211`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(2); rkey.PutString(`212`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(3); rkey.PutString(`301`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(3); rkey.PutString(`302`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(3); rkey.PutString(`303`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(3); rkey.PutString(`304`, 10); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`data` Custodian=`` Email=``> <IOSettings> <JobType>mapreduce</JobType> <KeyLength>nInt</KeyLength> <DFSInput>dfs://data_Input" + guid + @".txt</DFSInput> <DFSOutput>dfs://data_Output" + guid + @"4.bin@nInt,nChar(10)</DFSOutput> <OutputMethod>grouped</OutputMethod> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Using>RDBMS_DBCORE</Using> <MapReduce> <Map> <![CDATA[ public virtual void Map(ByteSlice line, MapOutput output) { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); output.Add(rkey.ToByteSlice(), ByteSlice.Prepare()); } ]]> </Map> <Reduce> <![CDATA[ public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output) { { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(3); rkey.PutString(`305`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); rkey.PutString(`401`, 10); output.Add(rkey.ToByteSlice()); } { DbRecordset rkey = DbRecordset.Prepare(); rkey.PutInt(4); rkey.PutString(`402`, 10); output.Add(rkey.ToByteSlice()); } } ]]> </Reduce> </MapReduce> </Job> <Job Name=`Prepare_tblFriends_Table_Create_RIndex` Custodian=`` Email=``> <IOSettings> <JobType>local</JobType> </IOSettings> <Add Reference=`RDBMS_DBCORE.dll` Type=`dfs`/> <Add Reference=`System.Data.dll` Type=`system`/> <Using>RDBMS_DBCORE</Using> <Using>System.Data</Using> <Using>System.Data.Common</Using> <Local> <![CDATA[ public virtual void Local() { Shell(@`qizmt combine data_Output" + guid + @"1.bin data_Output" + guid + @"2.bin data_Output" + guid + @"3.bin data_Output" + guid + @"4.bin +data_Output" + guid + @".bin`); System.Data.Common.DbProviderFactory fact = DbProviderFactories.GetFactory(`Qizmt_DataProvider`); using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create table " + tablename + @" (id int, rid char(10))`; cmd.ExecuteNonQuery(); cmd.CommandText = `insert into " + tablename + @" bind 'dfs://data_Output" + guid + @".bin'`; cmd.ExecuteNonQuery(); conn.Close(); } using (DbConnection conn = fact.CreateConnection()) { conn.ConnectionString = `Data Source = localhost`; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = `create rindex " + indexname + @" from " + tablename + @" pinmemoryHASH ON id `; cmd.ExecuteNonQuery(); conn.Close(); } } ]]> </Local> </Job> </Jobs> </SourceCode>").Replace('`', '"'); string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\" + Environment.CurrentDirectory.Replace(':', '$') + @"\" + Guid.NewGuid().ToString().Replace("-", ""); if (System.IO.Directory.Exists(tempdir)) { System.IO.Directory.Delete(tempdir, true); } System.IO.Directory.CreateDirectory(tempdir); string tempjobname = Guid.NewGuid().ToString(); System.IO.File.WriteAllText(tempdir + @"\" + tempjobname, job); Exec.Shell("Qizmt importdir \"" + tempdir + "\""); Exec.Shell("dspace exec " + tempjobname); //Clean up Exec.Shell(@"Qizmt del " + tempjobname); System.IO.Directory.Delete(tempdir, true); } { Console.WriteLine("RSelecting samples from rindex..."); DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = " + string.Join(",", allhosts) + "; rindex=pooled"; conn.Open(); DbCommand cmd = conn.CreateCommand(); //Getting something not in the table also. cmd.CommandText = "rselect * from " + indexname + " sample 5 where key=-888 or key = 1 or key = 2 or key=3 or key=4 or key=-9999"; DbDataReader reader = cmd.ExecuteReader(); Dictionary <int, List <string> > results = new Dictionary <int, List <string> >(); while (reader.Read()) { int x = reader.GetInt32(0); string y = reader.GetString(1); if (!results.ContainsKey(x)) { results.Add(x, new List <string>()); } results[x].Add(y); } reader.Close(); conn.Close(); Dictionary <int, Dictionary <string, int> > expected = new Dictionary <int, Dictionary <string, int> >(); expected.Add(1, new Dictionary <string, int>()); for (int i = 101; i <= 109; i++) { expected[1].Add(i.ToString(), 0); } expected.Add(2, new Dictionary <string, int>()); for (int i = 201; i <= 212; i++) { expected[2].Add(i.ToString(), 0); } expected.Add(3, new Dictionary <string, int>()); for (int i = 301; i <= 305; i++) { expected[3].Add(i.ToString(), 0); } expected.Add(4, new Dictionary <string, int>()); for (int i = 401; i <= 402; i++) { expected[4].Add(i.ToString(), 0); } //Make sure the correct number of samples are returned. Dictionary <int, int> expectedCount = new Dictionary <int, int>(); expectedCount.Add(1, 5); expectedCount.Add(2, 5); expectedCount.Add(3, 5); expectedCount.Add(4, 2); foreach (int key in results.Keys) { if (!expected.ContainsKey(key)) { throw new Exception("Unexpected key received: " + key.ToString()); } foreach (string val in results[key]) { if (!expected[key].ContainsKey(val)) { throw new Exception("The pair: (" + key.ToString() + ", '" + val + "') is not found."); } if (expected[key][val] != 0) { throw new Exception("The pair: (" + key.ToString() + ", '" + val + "') is returned more than once."); } expected[key][val]++; } if (results[key].Count != expectedCount[key]) { throw new Exception("Expected rows to be returned for key= " + key.ToString() + " is " + expectedCount[key].ToString() + ", but got " + results[key].Count.ToString() + " rows instead."); } } } { DbConnection conn = fact.CreateConnection(); conn.ConnectionString = "Data Source = localhost"; conn.Open(); DbCommand cmd = conn.CreateCommand(); cmd.CommandText = "drop table " + tablename; cmd.ExecuteNonQuery(); cmd.CommandText = "drop rindex " + indexname; cmd.ExecuteNonQuery(); conn.Close(); } }