void _SysTablesXML(System.IO.Stream output)
{
string xml;
using (GlobalCriticalSection.GetLock())
{
xml = LoadSysTablesXml_unlocked();
}
int ColLineLength = 1 + (200 * 2);
foreach (string _line in xml.Split('\n'))
{
string line = _line.Trim('\n', '\r');
recordset rs = recordset.Prepare();
rs.PutByte(0); // Nullable byte.
rs.PutString(line);
{
ByteSlice bs = rs.ToByteSlice();
if (bs.Length > ColLineLength)
{
throw new Exception("Column Line: length is too long: " + line);
}
for (int ibs = 0; ibs < bs.Length; ibs++)
{
output.WriteByte(bs[ibs]);
}
for (int ibs = bs.Length; ibs < DSpace_OutputRecordLength; ibs++)
{
output.WriteByte(0);
}
}
}
}
protected override void Run()
{
string TableName = DSpace_ExecArgs[0];
string ImportDfsFile = DSpace_ExecArgs[1];
string OutputDfsFile = DSpace_ExecArgs[2]; // Gets combined into DfsTableFile.
string QlArgDelim = DSpace_ExecArgs[3];
if (GetDfsFileRecordLength(ImportDfsFile) > 0)
{
throw new Exception("Import file '" + ImportDfsFile + "' not found or has incorrect type");
}
System.Xml.XmlDocument systables;
using (GlobalCriticalSection.GetLock())
{
systables = LoadSysTables_unlocked();
}
System.Xml.XmlElement xeTable = FindTable(systables, TableName);
if (null == xeTable)
{
throw new Exception("Table '" + TableName + "' does not exist");
}
string DfsTableFile = xeTable["file"].InnerText;
string RowInfo;
string TypeInfo; // Type
int ExpectedImportRowLength = 0;
int OutputRowLength = 0;
{
StringBuilder sbRowInfo = new StringBuilder();
StringBuilder sbTypeInfo = new StringBuilder(); // Type
foreach (System.Xml.XmlNode xn in xeTable.SelectNodes("column"))
{
if (0 != sbRowInfo.Length)
{
sbRowInfo.Append(',');
sbTypeInfo.Append(','); // Type
}
string stsize = xn["bytes"].InnerText;
int tsize = int.Parse(stsize);
ExpectedImportRowLength += tsize - 1; // Import rows don't include Nullable byte.
OutputRowLength += tsize;
sbRowInfo.Append(stsize);
sbTypeInfo.Append(xn["type"].InnerText); // Type
}
RowInfo = sbRowInfo.ToString();
TypeInfo = sbTypeInfo.ToString(); // Type
}
string overrideFTE = "";
if (RDBMS_DBCORE.Qa.FaultTolerantExecution)
{
overrideFTE = " \"//Job[@Name='RDBMS_ImportLines']/FaultTolerantExecution/Mode=enabled\" ";
}
DSpace_Log(Shell("dspace exec \"//Job[@Name='RDBMS_ImportLines']/IOSettings/DFSInput=" + ImportDfsFile + "\" \"//Job[@Name='RDBMS_ImportLines']/IOSettings/DFSOutput=" + OutputDfsFile + "@" + OutputRowLength.ToString() + "\" " + overrideFTE + " RDBMS_ImportLines.DBCORE \"" + TableName + "\" \"" + ImportDfsFile + "\" \"" + OutputDfsFile + "\" \"" + DfsTableFile + "\" \"" + RowInfo + "\" \"" + TypeInfo + "\" \"" + QlArgDelim + "\"").Trim());
}
protected override void Run()
{
string TableName = DSpace_ExecArgs[0];
string OutputDfsFile = DSpace_ExecArgs[1]; // Gets combined into DfsTableFile.
string QlArgVals = DSpace_ExecArgs[2];
System.Xml.XmlDocument systables;
using (GlobalCriticalSection.GetLock())
{
systables = LoadSysTables_unlocked();
}
System.Xml.XmlElement xeTable = FindTable(systables, TableName);
if (null == xeTable)
{
throw new Exception("Table '" + TableName + "' does not exist");
}
string DfsTableFile = xeTable["file"].InnerText;
string RowInfo;
string TypeInfo; // Type
int OutputRowLength = 0;
{
StringBuilder sbRowInfo = new StringBuilder();
StringBuilder sbTypeInfo = new StringBuilder(); // Type
foreach (System.Xml.XmlNode xn in xeTable.SelectNodes("column"))
{
if (0 != sbRowInfo.Length)
{
sbRowInfo.Append(',');
sbTypeInfo.Append(','); // Type
}
string stsize = xn["bytes"].InnerText;
int tsize = int.Parse(stsize);
OutputRowLength += tsize;
sbRowInfo.Append(stsize);
sbTypeInfo.Append(xn["type"].InnerText); // Type
}
RowInfo = sbRowInfo.ToString();
TypeInfo = sbTypeInfo.ToString(); // Type
}
DSpace_Log(Shell("dspace exec \"//Job[@Name='RDBMS_InsertValues']/IOSettings/DFS_IO/DFSWriter=" + OutputDfsFile + "@" + OutputRowLength.ToString() + "\" RDBMS_InsertValues.DBCORE \"" + TableName + "\" \"" + OutputDfsFile + "\" \"" + QlArgVals + "\" \"" + DfsTableFile + "\" \"" + RowInfo + "\" \"" + TypeInfo + "\"").Trim());
}
void _SysTables(System.IO.Stream output)
{
System.Xml.XmlDocument systables;
using (GlobalCriticalSection.GetLock())
{
systables = LoadSysTables_unlocked();
}
foreach (System.Xml.XmlNode xn in systables.SelectNodes("/tables/table"))
{
recordset rs = recordset.Prepare();
int ColTableLength = 1 + (100 * 2);
string Table = xn["name"].InnerText;
rs.PutByte(0); // Nullable byte.
rs.PutString(Table);
{
ByteSlice bs = rs.ToByteSlice();
if (bs.Length > ColTableLength)
{
throw new Exception("Column Table: length is too long: " + Table);
}
for (int ibs = bs.Length; ibs < ColTableLength; ibs++)
{
rs.PutByte(0);
}
}
int ColFileLength = 1 + (120 * 2);
string File = xn["file"].InnerText;
rs.PutByte(0); // Nullable byte.
rs.PutString(File);
{
ByteSlice bs = rs.ToByteSlice();
if (bs.Length > ColTableLength + ColFileLength)
{
throw new Exception("Column File: length is too long: " + File);
}
for (int ibs = bs.Length; ibs < ColTableLength + ColFileLength; ibs++)
{
rs.PutByte(0);
}
}
{
ByteSlice bs = rs.ToByteSlice();
if (bs.Length > DSpace_OutputRecordLength)
{
throw new Exception("Record too long");
}
for (int ibs = 0; ibs < bs.Length; ibs++)
{
output.WriteByte(bs[ibs]);
}
for (int ibs = bs.Length; ibs < DSpace_OutputRecordLength; ibs++)
{
output.WriteByte(0);
}
}
}
}
protected override void Run()
{
string QlLeftTableName = DSpace_ExecArgs[0];
string LeftTableName = QlArgsUnescape(QlLeftTableName);
string stype = DSpace_ExecArgs[1];
string QlRightTableName = DSpace_ExecArgs[2];
string RightTableName = QlArgsUnescape(QlRightTableName);
string QlOn = DSpace_ExecArgs[3];
string On = QlArgsUnescape(QlOn);
if (-1 != LeftTableName.IndexOf('\0'))
{
throw new NotSupportedException("Cannot JOIN with multiple left tables: " + LeftTableName);
}
if (-1 != RightTableName.IndexOf('\0'))
{
throw new NotSupportedException("Cannot JOIN with multiple right tables: " + RightTableName);
}
System.Xml.XmlDocument systables;
using (GlobalCriticalSection.GetLock())
{
systables = LoadSysTables_unlocked();
}
System.Xml.XmlElement xeLeftTable = FindTable(systables, LeftTableName);
if (null == xeLeftTable)
{
throw new Exception("Table (left) '" + LeftTableName + "' does not exist");
}
string sLeftRowSize = xeLeftTable["size"].InnerText;
int LeftRowSize = int.Parse(sLeftRowSize);
string LeftDfsTableFilesInput = xeLeftTable["file"].InnerText + "@" + sLeftRowSize;
if (LeftDfsTableFilesInput.StartsWith("qa://", true, null))
{
throw new Exception("Cannot JOIN with system tables (left)");
}
System.Xml.XmlElement xeRightTable = FindTable(systables, RightTableName);
if (null == xeRightTable)
{
throw new Exception("Table (right) '" + RightTableName + "' does not exist");
}
string sRightRowSize = xeRightTable["size"].InnerText;
int RightRowSize = int.Parse(sRightRowSize);
string RightDfsTableFilesInput = xeRightTable["file"].InnerText + "@" + sRightRowSize;
if (RightDfsTableFilesInput.StartsWith("qa://", true, null))
{
throw new Exception("Cannot JOIN with system tables (right)");
}
string DfsTableFilesInput = LeftDfsTableFilesInput + ";" + RightDfsTableFilesInput;
string DfsOutputName = "dfs://RDBMS_JoinOn_" + Guid.NewGuid().ToString() + Qa.DFS_TEMP_FILE_MARKER;
int DfsOutputRowSize = LeftRowSize + RightRowSize;
string LeftRowInfo;
string LeftDisplayInfo; // Display
List <DbColumn> LeftCols = new List <DbColumn>();
List <string> LeftColsWidths = new List <string>();
{
StringBuilder sbRowInfo = new StringBuilder();
StringBuilder sbDisplayInfo = new StringBuilder(); // Display
int totsize = 0;
string xtablename = xeLeftTable["name"].InnerText;
foreach (System.Xml.XmlNode xn in xeLeftTable.SelectNodes("column"))
{
if (0 != sbRowInfo.Length)
{
sbRowInfo.Append('\0');
sbDisplayInfo.Append(','); // Display
}
string stsize = xn["bytes"].InnerText;
int tsize = int.Parse(stsize);
string RealColName = xn["name"].InnerText;
string UserColName = RealColName;
string xcolname;
if (-1 == UserColName.IndexOf('.'))
{
xcolname = xtablename + "." + UserColName;
}
else
{
xcolname = UserColName;
}
sbRowInfo.Append(xcolname); // Note: doesn't consider sub-select.
sbRowInfo.Append('=');
sbRowInfo.Append(stsize);
sbDisplayInfo.Append(xn["type"].InnerText); // Display
sbDisplayInfo.Append('='); // Display
sbDisplayInfo.Append(xn["dw"].InnerText); // Display
LeftColsWidths.Add(xn["dw"].InnerText);
{
DbColumn c;
c.Type = DbType.Prepare(xn["type"].InnerText, tsize);
c.RowOffset = totsize;
c.ColumnName = xcolname;
LeftCols.Add(c);
}
totsize += tsize;
}
LeftRowInfo = sbRowInfo.ToString();
LeftDisplayInfo = sbDisplayInfo.ToString(); // Display
}
string RightRowInfo;
string RightDisplayInfo; // Display
List <DbColumn> RightCols = new List <DbColumn>();
List <string> RightColsWidths = new List <string>();
{
StringBuilder sbRowInfo = new StringBuilder();
StringBuilder sbDisplayInfo = new StringBuilder(); // Display
int totsize = 0;
string xtablename = xeRightTable["name"].InnerText;
foreach (System.Xml.XmlNode xn in xeRightTable.SelectNodes("column"))
{
if (0 != sbRowInfo.Length)
{
sbRowInfo.Append('\0');
sbDisplayInfo.Append(','); // Display
}
string stsize = xn["bytes"].InnerText;
int tsize = int.Parse(stsize);
string RealColName = xn["name"].InnerText;
string UserColName = RealColName;
string xcolname;
if (-1 == UserColName.IndexOf('.'))
{
xcolname = xtablename + "." + UserColName;
}
else
{
xcolname = UserColName;
}
sbRowInfo.Append(xcolname); // Note: doesn't consider sub-select.
sbRowInfo.Append('=');
sbRowInfo.Append(stsize);
sbDisplayInfo.Append(xn["type"].InnerText); // Display
sbDisplayInfo.Append('='); // Display
sbDisplayInfo.Append(xn["dw"].InnerText); // Display
RightColsWidths.Add(xn["dw"].InnerText);
{
DbColumn c;
c.Type = DbType.Prepare(xn["type"].InnerText, tsize);
c.RowOffset = totsize;
c.ColumnName = xcolname;
RightCols.Add(c);
}
totsize += tsize;
}
RightRowInfo = sbRowInfo.ToString();
RightDisplayInfo = sbDisplayInfo.ToString(); // Display
}
string on1, onop, on2;
{
string onargs = On;
on1 = Qa.NextPart(ref onargs);
onop = Qa.NextPart(ref onargs);
on2 = Qa.NextPart(ref onargs);
if (0 == on1.Length ||
0 == onop.Length ||
0 == on2.Length ||
0 != onargs.Trim().Length)
{
throw new Exception("Invalid ON expression for JOIN: " + On);
}
}
bool on1left;
int on1colindex;
{
int on1colindexother;
on1colindex = DbColumn.IndexOf(LeftCols, on1);
on1colindexother = DbColumn.IndexOf(RightCols, on1);
if (-1 != on1colindex)
{
if (-1 != on1colindexother)
{
throw new Exception("Column name " + on1 + " does not resolve to a single column (in left and right tables)");
}
else
{
on1left = true;
}
}
else
{
if (-1 != on1colindexother)
{
on1colindex = on1colindexother;
on1left = false;
}
else
{
throw new Exception("No such column named " + on1);
}
}
}
bool on2left;
int on2colindex;
{
int on2colindexother;
on2colindex = DbColumn.IndexOf(LeftCols, on2);
on2colindexother = DbColumn.IndexOf(RightCols, on2);
if (-1 != on2colindex)
{
if (-1 != on2colindexother)
{
throw new Exception("Column name " + on2 + " does not resolve to a single column (in left and right tables)");
}
else
{
on2left = true;
}
}
else
{
if (-1 != on2colindexother)
{
on2colindex = on2colindexother;
on2left = false;
}
else
{
throw new Exception("No such column named " + on2);
}
}
}
if ((on1left && on2left) ||
(!on1left && !on2left))
{
string whicht;
if (on1left)
{
whicht = "(left) " + LeftTableName;
}
else
{
whicht = "(right) " + RightTableName;
}
throw new Exception("ON expression is comparing columns from the same table: "
+ on1 + " and " + on2 + " are both part of " + whicht);
}
// Order ON columns:
if (!on1left)
{
{
string onx = on1;
on1 = on2;
on2 = onx;
}
{
int onxcolindex = on1colindex;
on1colindex = on2colindex;
on2colindex = onxcolindex;
}
{
on1left = true;
on2left = false;
}
// Invert the operator..
switch (onop)
{
case "=":
//onop = "=";
break;
case "!=":
//onop = "!=";
break;
case "<":
onop = ">";
break;
case "<=":
onop = ">=";
break;
case ">":
onop = "<";
break;
case ">=":
onop = "<=";
break;
default:
throw new Exception("Unhandled ON operation: " + onop);
}
On = on1 + " " + onop + " " + on2;
QlOn = QlArgsEscape(On);
}
DbColumn on1col;
if (on1left)
{
on1col = LeftCols[on1colindex];
}
else
{
//on1col = RightCols[on1colindex];
throw new Exception("DEBUG: PrepareJoinOn: (!on1left)");
}
if (on1col.Type.Size == 0 || on1col.Type.ID == DbTypeID.NULL)
{
throw new Exception("Invalid column for ON expression: " + on1);
}
DbColumn on2col;
if (on2left)
{
//on2col = LeftCols[on2colindex];
throw new Exception("DEBUG: PrepareJoinOn: (on2left)");
}
else
{
on2col = RightCols[on2colindex];
}
if (on2col.Type.Size == 0 || on2col.Type.ID == DbTypeID.NULL)
{
throw new Exception("Invalid column for ON expression: " + on2);
}
int KeyLength = on1col.Type.Size;
if (on2col.Type.Size > KeyLength)
{
KeyLength = on2col.Type.Size;
}
{
MapReduceCall mrc = GetMapReduceCallJoinOn(DfsTableFilesInput);
mrc.OverrideOutputMethod = "grouped";
mrc.OverrideInput = DfsTableFilesInput;
mrc.OverrideOutput = DfsOutputName + "@" + DfsOutputRowSize;
mrc.OverrideKeyLength = KeyLength;
if (RDBMS_DBCORE.Qa.FaultTolerantExecution)
{
mrc.OverrideFaultTolerantExecutionMode = "enabled";
}
mrc.Call("\"" + QlLeftTableName + "\" " + stype + " \"" + QlRightTableName
+ "\" \"" + QlOn
+ "\" \"" + on1col.RowOffset + "," + on1col.Type.Size + "=" + QlArgsEscape(on1col.Type.Name)
+ "\" \"" + on2col.RowOffset + "," + on2col.Type.Size + "=" + QlArgsEscape(on2col.Type.Name)
+ "\" \"" + DfsTableFilesInput
+ "\"");
}
{
PrepareSelect.queryresults qr = new PrepareSelect.queryresults();
List <DbColumn> AllCols = new List <DbColumn>(LeftCols.Count + RightCols.Count);
AllCols.AddRange(LeftCols);
AllCols.AddRange(RightCols);
qr.SetFields(AllCols);
qr.temptable = DfsOutputName;
qr.recordsize = LeftRowSize + RightRowSize;
string sPartCount = Shell("dspace countparts \"" + DfsOutputName + "\"").Split('\n')[0].Trim();
qr.parts = int.Parse(sPartCount);
string sDfsOutputSize = Shell("dspace filesize \"" + DfsOutputName + "\"").Split('\n')[0].Trim();
long DfsOutputSize = long.Parse(sDfsOutputSize);
long NumRowsOutput = DfsOutputSize / qr.recordsize;
qr.recordcount = NumRowsOutput;
DSpace_Log(PrepareSelect.SetQueryResults(qr));
}
}