protected override void Run() { #if DEBUG //System.Diagnostics.Debugger.Launch(); #endif string TableName = QlArgsUnescape(DSpace_ExecArgs[0]); // nul-delimited. string DfsOutputName = DSpace_ExecArgs[1]; string QlArgsSelectWhat = DSpace_ExecArgs[2]; long TopCount = long.Parse(DSpace_ExecArgs[3]); string QlArgsOps = DSpace_ExecArgs[4]; // Still encoded with QlArgsEscape. string sOptions = DSpace_ExecArgs[5]; // "DFSTEMP" or "-", etc bool dfstemp = -1 != sOptions.IndexOf("DFSTEMP"); bool Update = -1 != sOptions.IndexOf("GUPDATE"); // UPDATE (grouped MR). bool GroupBy = -1 != sOptions.IndexOf("GBY"); bool OrderBy = -1 != sOptions.IndexOf("OBY"); bool Order2By = -1 != sOptions.IndexOf("O2BY"); bool dfsref = -1 != sOptions.IndexOf("DFSREF"); bool queryresults = dfstemp || dfsref; bool topdfstemp = -1 != sOptions.IndexOf("TOPTEMP"); bool distinct = -1 != sOptions.IndexOf("DISTINCT"); bool joined = -1 != sOptions.IndexOf("JOINED"); #if DEBUG if (TopCount == 4242) { System.Diagnostics.Debugger.Launch(); } #endif string SelectWhat = QlArgsUnescape(QlArgsSelectWhat); bool WhatFunctions = -1 != SelectWhat.IndexOf('('); // Select clause has functions (aggregates and/or scalars). if (Order2By) { OrderBy = true; WhatFunctions = false; #if DEBUG if (GroupBy) { throw new Exception("DEBUG: Not supposed to find GroupBy here with Order2By"); } #endif #if DEBUG //System.Diagnostics.Debugger.Launch(); #endif } bool HasKeyOn = OrderBy || GroupBy; if (OrderBy) { if (WhatFunctions || GroupBy) { if ("*" == SelectWhat) { throw new Exception("Invalid query: cannot SELECT * with ORDER BY and GROUP BY"); } string sOrderByCols = null; // null, or "foo,bar" from "ORDER BY foo,bar". List<string> OrderByCols = new List<string>(); string[] Ops = QlArgsUnescape(QlArgsOps).Split('\0'); List<string> NewOps = new List<string>(Ops.Length); for (int iop = 0; iop < Ops.Length; iop++) { if (0 == string.Compare("ORDER", Ops[iop], true)) { if (iop + 1 < Ops.Length && 0 == string.Compare("BY", Ops[iop + 1], true)) { iop++; int nparens = 0; StringBuilder sob = new StringBuilder(); StringBuilder curobcol = new StringBuilder(); for (; ; ) { iop++; if (iop >= Ops.Length) { break; } sob.Append(Ops[iop]); curobcol.Append(Ops[iop]); if ("(" == Ops[iop]) { nparens++; } else if (nparens > 0) { if (")" == Ops[iop]) { nparens--; } } iop++; if (iop >= Ops.Length) { if (0 != nparens) { throw new Exception("Expected ) in ORDER BY"); } if (curobcol.Length != 0) { OrderByCols.Add(curobcol.ToString()); curobcol.Length = 0; } break; } if (0 == nparens) { if (curobcol.Length != 0) { OrderByCols.Add(curobcol.ToString()); curobcol.Length = 0; } if ("," != Ops[iop]) { break; } } else { curobcol.Append(Ops[iop]); } sob.Append(Ops[iop]); } sOrderByCols = sob.ToString(); if (iop >= Ops.Length) { break; } } } NewOps.Add(Ops[iop]); } QlArgsOps = QlArgsEscape(string.Join("\0", NewOps.ToArray())); #if DEBUG //System.Diagnostics.Debugger.Launch(); #endif QueryAnalyzer qa1 = new QueryAnalyzer(); string sel1 = qa1.Exec( "SELECT DFSTEMP", QlArgsEscape(sOrderByCols + "," + SelectWhat.Replace('\0', ',')), "FROM", QlArgsEscape(TableName), QlArgsEscape(string.Join(" ", NewOps.ToArray())) ); { if (-1 != TableName.IndexOf(Qa.DFS_TEMP_FILE_MARKER)) { try { System.Xml.XmlElement tt = GetDfsTableInfo(TableName); string ttf = tt["file"].InnerText; if (-1 != ttf.IndexOf(Qa.DFS_TEMP_FILE_MARKER)) { dfsclient.DeleteFile(ttf); } } catch { } } } Qa.PrepareSelect.queryresults qr = Qa.PrepareSelect.GetQueryResults(sel1); for (int iobc = 0; iobc < OrderByCols.Count; iobc++) { qr.fields[iobc].name = "~OBY.~" + qr.fields[iobc].name; } try { string newopts = ""; if (dfstemp) { newopts += ";DFSTEMP"; } if (dfsref) { newopts += ";DFSREF"; } if (topdfstemp) { newopts += ";TOPTEMP"; } if (distinct) { newopts += ";DISTINCT"; } PrepareSelect ps2 = new PrepareSelect(); string neworderby; { StringBuilder sbnob = new StringBuilder(); //"ORDER\0BY\0" + sOrderByCols.Replace(',', '\0') sbnob.Append("ORDER\0BY"); bool firstsbnob = true; for (int iobc = 0; iobc < OrderByCols.Count; iobc++) { if (!firstsbnob) { sbnob.Append("\0,"); } firstsbnob = false; sbnob.Append('\0'); sbnob.Append(qr.fields[iobc].name); // Includes "~OBY.~" } neworderby = sbnob.ToString(); } #if DEBUG //System.Diagnostics.Debugger.Launch(); #endif newopts += ";O2BY"; // Order-by phase 2. // [QlArgs]SelectWhat might have () but they are NOT evaluated at this point (no SFUNC) string sel2 = ps2.Exec( Qa.QlArgsEscape(qr.GetPseudoTableName()), DfsOutputName, //(OrderByCols.Split(',').Length + 2).ToString() + "-*", // Select N-* where N is after OrderByCols QlArgsSelectWhat, TopCount.ToString(), QlArgsEscape(neworderby), newopts ); DSpace_Log(sel2); } finally { if (qr.IsTempTable) { { string delfile = qr.temptable; { int iat = delfile.IndexOf('@'); if (-1 != iat) { delfile = delfile.Substring(0, iat); } } dfsclient.DeleteFile(delfile); } } } return; } } string[] awhat = null; string[] UserColumnNames = null; if ("*" != SelectWhat && "^" != SelectWhat) { awhat = SelectWhat.Split('\0'); UserColumnNames = new string[awhat.Length]; for (int iww = 0; iww < awhat.Length; iww++) { int ilas = awhat[iww].LastIndexOf(" AS ", StringComparison.OrdinalIgnoreCase); if (-1 == ilas) { UserColumnNames[iww] = awhat[iww]; } else { string asname = awhat[iww].Substring(ilas + 4); for (int iy = 0; iy < asname.Length; iy++) { if (!char.IsLetterOrDigit(asname[iy]) && '_' != asname[iy]) { asname = null; break; } } if (!string.IsNullOrEmpty(asname) && !char.IsDigit(asname[0])) { UserColumnNames[iww] = asname; awhat[iww] = awhat[iww].Substring(0, ilas); } else { UserColumnNames[iww] = awhat[iww]; } } } } #if DEBUG if (Update && !dfstemp) { throw new Exception("DEBUG: (Update && !dfstemp)"); } #endif System.Xml.XmlDocument systables; using (GlobalCriticalSection.GetLock()) { systables = LoadSysTables_unlocked(); } System.Xml.XmlElement xeTable; string DfsTableFilesInput; // Note: can be multiple semicolon-separated input files. Includes record length info (@N). string sRowSize; int RowSize; if (-1 != TableName.IndexOf('\0')) { string[] tables = TableName.Split('\0'); string tn = tables[0]; xeTable = FindTable(systables, tn); if (null == xeTable) { throw new Exception("Table '" + tn + "' does not exist"); } sRowSize = xeTable["size"].InnerText; RowSize = int.Parse(sRowSize); DfsTableFilesInput = xeTable["file"].InnerText + "@" + sRowSize; System.Xml.XmlNodeList xnlTableCols = xeTable.SelectNodes("column"); int numcols = xnlTableCols.Count; for (int tni = 1; tni < tables.Length; tni++) { tn = tables[tni]; System.Xml.XmlElement xett = FindTable(systables, tn); if (null == xett) { throw new Exception("Table '" + tn + "' does not exist"); } System.Xml.XmlNodeList xnltcols = xett.SelectNodes("column"); bool colsmatch = false; if (xnltcols.Count == numcols) { bool fail = false; for (int ic = 0; ic < numcols; ic++) { if (xnlTableCols[ic]["type"].InnerText != xnltcols[ic]["type"].InnerText) { fail = true; break; } if (0 != string.Compare(xnlTableCols[ic]["name"].InnerText, xnltcols[ic]["name"].InnerText, true)) { fail = true; break; } } colsmatch = !fail; } if (!colsmatch) { throw new Exception("Columns of table " + tn + " do not match columns of table " + xeTable["name"].InnerText); } DfsTableFilesInput += ";" + xett["file"].InnerText + "@" + sRowSize; } if (-1 != DfsTableFilesInput.IndexOf("qa://", StringComparison.OrdinalIgnoreCase)) { throw new Exception("Cannot union with a system table"); ///////// } } else { xeTable = FindTable(systables, TableName); if (null == xeTable) { throw new Exception("Table '" + TableName + "' does not exist"); } sRowSize = xeTable["size"].InnerText; RowSize = int.Parse(sRowSize); DfsTableFilesInput = xeTable["file"].InnerText + "@" + sRowSize; } if (dfsref) { DfsOutputName = xeTable["file"].InnerText; } if (queryresults) { DSpace_Log("<?xml version=\"1.0\"?>"); DSpace_Log(" <queryresults>"); if (dfsref) { DSpace_Log(" <reftable>" + DfsOutputName + "</reftable>"); } else { DSpace_Log(" <temptable>" + DfsOutputName + "</temptable>"); } } string RowInfo; string DisplayInfo; // Display cols = new List<DbColumn>(); colswidths = new List<string>(); { StringBuilder sbRowInfo = new StringBuilder(); StringBuilder sbDisplayInfo = new StringBuilder(); // Display int totsize = 0; string xtablename = xeTable["name"].InnerText; foreach (System.Xml.XmlNode xn in xeTable.SelectNodes("column")) { if (0 != sbRowInfo.Length) { sbRowInfo.Append('\0'); sbDisplayInfo.Append(','); // Display } string stsize = xn["bytes"].InnerText; int tsize = int.Parse(stsize); string RealColName = xn["name"].InnerText; string UserColName = RealColName; if (null != awhat) { for (int iww = 0; iww < awhat.Length; iww++) { if (0 == string.Compare(awhat[iww], RealColName, true)) { UserColName = UserColumnNames[iww]; break; } } } string xcolname; if (-1 == UserColName.IndexOf('.')) { xcolname = xtablename + "." + UserColName; } else { xcolname = UserColName; } sbRowInfo.Append(xcolname); // Note: doesn't consider sub-select. sbRowInfo.Append('='); sbRowInfo.Append(stsize); sbDisplayInfo.Append(xn["type"].InnerText); // Display sbDisplayInfo.Append('='); // Display sbDisplayInfo.Append(xn["dw"].InnerText); // Display colswidths.Add(xn["dw"].InnerText); { DbColumn c; c.Type = DbType.Prepare(xn["type"].InnerText, tsize); c.RowOffset = totsize; c.ColumnName = xcolname; cols.Add(c); } totsize += tsize; } RowInfo = sbRowInfo.ToString(); DisplayInfo = sbDisplayInfo.ToString(); // Display } int KeyLength; if (HasKeyOn) { KeyLength = RowSize; } else { KeyLength = 4; } bool IsSpecialOrdered = -1 != sOptions.IndexOf("SPECIALORDER"); string DfsInput; string DfsTempInputFile = null; // FIXME: look for all qa:// and run them through separately,... if (DfsTableFilesInput.StartsWith("qa://", true, null)) { if (dfsref) { throw new Exception("Cannot DFSREF with non-user table"); } string SysGenOutputFile; if (IsSpecialOrdered) { SysGenOutputFile = DfsOutputName + "@" + sRowSize; DfsInput = null; } else { SysGenOutputFile = "dfs://RDBMS_QaTemp_" + Guid.NewGuid().ToString() + "@" + sRowSize; DfsTempInputFile = SysGenOutputFile; DfsInput = DfsTempInputFile; } { string qafile = DfsTableFilesInput; int iat = qafile.IndexOf('@'); if (-1 != iat) { qafile = qafile.Substring(0, iat); } { RemoteCall rc = GetRemoteCallSysGen(); //rc.OverrideInput = #; rc.OverrideOutput = SysGenOutputFile; //rc.OverrideKeyLength = #; rc.Call("\"" + qafile + "\" \"" + Qa.QlArgsEscape(RowInfo) + "\" \"" + DisplayInfo + "\" \"" + Qa.QlArgsEscape(TableName) + "\"").Trim(); } } } else { DfsInput = DfsTableFilesInput; } #if DEBUG if (null != DfsInput) { if (-1 == DfsInput.IndexOf('@')) { throw new Exception("Expected @ in DfsInput"); } } #endif string OutputRowInfo; string OutputDisplayInfo; long OutputRowSize; string OutputsRowSize; List<string> outputcolswidths; List<DbColumn> outputcols; if (null != awhat) { outputcols = new List<DbColumn>(awhat.Length); outputcolswidths = new List<string>(); StringBuilder sbRowInfo = new StringBuilder(); StringBuilder sbDisplayInfo = new StringBuilder(); long xRowSize = 0; for (int iww = 0; iww < UserColumnNames.Length; iww++) { string w = UserColumnNames[iww]; string sdw; // String display width. DbColumn c = GetDbColumn(w, out sdw); if (c.Type.Size == 0) { throw new Exception("No such column named " + w); } if (0 != sbRowInfo.Length) { sbRowInfo.Append('\0'); sbDisplayInfo.Append(','); } { outputcols.Add(c); } //sbRowInfo.Append(c.ColumnName); // Already includes "TableName." sbRowInfo.Append(UserColumnNames[iww]); sbRowInfo.Append('='); sbRowInfo.Append(c.Type.Size); sbDisplayInfo.Append(c.Type.Name); sbDisplayInfo.Append('='); sbDisplayInfo.Append(sdw); xRowSize += c.Type.Size; outputcolswidths.Add(sdw); } OutputRowInfo = sbRowInfo.ToString(); OutputDisplayInfo = sbDisplayInfo.ToString(); OutputRowSize = xRowSize; OutputsRowSize = xRowSize.ToString(); } else { // Same values! outputcols = new List<DbColumn>(cols); OutputRowInfo = RowInfo; OutputDisplayInfo = DisplayInfo; OutputRowSize = RowSize; OutputsRowSize = sRowSize; { outputcolswidths = new List<string>(colswidths.Count); for (int icw = 0; icw < colswidths.Count; icw++) { outputcolswidths.Add(colswidths[icw]); } } } string QlArgsNewSelectWhat = QlArgsSelectWhat; if (null != UserColumnNames) { QlArgsNewSelectWhat = QlArgsEscape(string.Join("\0", UserColumnNames)); } string shelloutputSelect1 = ""; if (!dfsref) // Important! Don't run Select DBCORE if DFSREF. { if (!IsSpecialOrdered) { if (WhatFunctions) { string FuncDfsInput = DfsInput; string FuncDfsOutput = "dfs://RDBMS_SelectFunc_" + Guid.NewGuid().ToString(); FuncDfsOutput = DfsOutputName; // For now... string FuncArgsOptions = "SFUNC"; // Select clause functions (aggregates and/or scalars). if (GroupBy) { FuncArgsOptions += ";GBY"; } string FuncSelectOutput1; { MapReduceCall mrc = GetMapReduceCallSelect(FuncDfsInput); mrc.OverrideOutputMethod = "grouped"; mrc.OverrideInput = FuncDfsInput; mrc.OverrideOutput = FuncDfsOutput + "@" + OutputsRowSize; mrc.OverrideKeyLength = KeyLength; FuncSelectOutput1 = mrc.Call("\"" + TableName + "\" \"" + DfsOutputName + "\" \"" + QlArgsNewSelectWhat + "\" " + TopCount.ToString() + " \"" + QlArgsOps + "\" \"" + Qa.QlArgsEscape(RowInfo) + "\" \"" + DisplayInfo + "\" \"" + Qa.QlArgsEscape(OutputRowInfo) + "\" " + FuncArgsOptions).Trim(); } string[] FuncOutputTypeNames; { const string AOTIBEGINSTRING = "BEGIN:{AC596AA3-8E2F-41fa-B9E1-601D92F08AEC}"; int aotiBegin = FuncSelectOutput1.IndexOf(AOTIBEGINSTRING); if (-1 == aotiBegin) { string et = "Function (aggregate and/or scalar) Select MR output invalid (expected begin output type information)"; //#if DEBUG et += "\r\nMR output:\r\n" + FuncSelectOutput1 + "\r\n"; //#endif throw new Exception(et); } int aotiEnd = FuncSelectOutput1.IndexOf("{AC596AA3-8E2F-41fa-B9E1-601D92F08AEC}:END"); if (aotiEnd < aotiBegin) { throw new Exception("Function (aggregate and/or scalar) Select MR output invalid (expected end output type information)"); } { string stypes = FuncSelectOutput1.Substring(aotiBegin + AOTIBEGINSTRING.Length, aotiEnd - aotiBegin - AOTIBEGINSTRING.Length); FuncOutputTypeNames = System.Text.RegularExpressions.Regex.Split(stypes, @"\{264E73F6-E3C9-43de-A3FD-9AC36F905087\}"); } } if (FuncOutputTypeNames.Length != outputcolswidths.Count) { throw new Exception("DEBUG: (FuncOutputTypeNames.Length != outputcolswidths.Count)"); } { StringBuilder sbOutputDisplayInfo = new StringBuilder(); for (int icw = 0; icw < FuncOutputTypeNames.Length; icw++) { if (icw > 0) { sbOutputDisplayInfo.Append(','); } sbOutputDisplayInfo.Append(FuncOutputTypeNames[icw]); sbOutputDisplayInfo.Append('='); sbOutputDisplayInfo.Append(outputcolswidths[icw]); } OutputDisplayInfo = sbOutputDisplayInfo.ToString(); } if (FuncOutputTypeNames.Length != outputcols.Count) { throw new Exception("DEBUG: (FuncOutputTypeNames.Length != outputcols.Count)"); } { // Fix output type, since I didn't know until the output was generated. for (int oic = 0; oic < FuncOutputTypeNames.Length; oic++) { DbColumn c = outputcols[oic]; c.Type = DbType.Prepare(FuncOutputTypeNames[oic], c.Type.Size); outputcols[oic] = c; } } DfsInput = FuncDfsOutput + "@" + OutputsRowSize; { RowInfo = OutputRowInfo; DisplayInfo = OutputDisplayInfo; QlArgsNewSelectWhat = "*"; } } else if (GroupBy) { string GByDfsInput = DfsInput; string GByDfsOutput = "dfs://RDBMS_SelectGroupBy_" + Guid.NewGuid().ToString(); GByDfsOutput = DfsOutputName; // For now... string GByArgsOptions = "GBY"; { MapReduceCall mrc = GetMapReduceCallSelect(GByDfsInput); mrc.OverrideOutputMethod = "grouped"; mrc.OverrideInput = GByDfsInput; mrc.OverrideOutput = GByDfsOutput + "@" + OutputsRowSize; mrc.OverrideKeyLength = KeyLength; if (RDBMS_DBCORE.Qa.FaultTolerantExecution) { mrc.OverrideFaultTolerantExecutionMode = "enabled"; } mrc.Call("\"" + TableName + "\" \"" + DfsOutputName + "\" \"" + QlArgsNewSelectWhat + "\" " + TopCount.ToString() + " \"" + QlArgsOps + "\" \"" + Qa.QlArgsEscape(RowInfo) + "\" \"" + DisplayInfo + "\" \"" + Qa.QlArgsEscape(OutputRowInfo) + "\" " + GByArgsOptions).Trim(); } DfsInput = GByDfsOutput + "@" + OutputsRowSize; { RowInfo = OutputRowInfo; DisplayInfo = OutputDisplayInfo; QlArgsNewSelectWhat = "*"; } } else { { MapReduceCall mrc = GetMapReduceCallSelect(DfsInput); if (!OrderBy || Update) { mrc.OverrideOutputMethod = "grouped"; if (RDBMS_DBCORE.Qa.FaultTolerantExecution) { mrc.OverrideFaultTolerantExecutionMode = "enabled"; } } mrc.OverrideInput = DfsInput; mrc.OverrideOutput = DfsOutputName + "@" + OutputsRowSize; mrc.OverrideKeyLength = KeyLength; shelloutputSelect1 = mrc.Call("\"" + TableName + "\" \"" + DfsOutputName + "\" \"" + QlArgsNewSelectWhat + "\" " + TopCount.ToString() + " \"" + QlArgsOps + "\" \"" + Qa.QlArgsEscape(RowInfo) + "\" \"" + DisplayInfo + "\" \"" + Qa.QlArgsEscape(OutputRowInfo) + "\"").Trim(); } } } else { if (null != awhat) { // This case shouldn't happen anyways.. becuase if custom columns, it's not a even special order command anymore. throw new Exception("Special order commands must select all columns: SELECT * FROM " + TableName + " ..."); } } } if (null != DfsTempInputFile) { { string delfile = DfsTempInputFile; { int iat = delfile.IndexOf('@'); if (-1 != iat) { delfile = delfile.Substring(0, iat); } } dfsclient.DeleteFile(delfile); } } if (distinct) { string outtablefn = DfsOutputName + "_out_" + Guid.NewGuid().ToString(); { MapReduceCall mrc = GetMapReduceCallDistinct(); //mrc.OverrideOutputMethod = #; mrc.OverrideInput = DfsOutputName + "@" + OutputsRowSize; mrc.OverrideOutput = outtablefn + "@" + OutputsRowSize; mrc.OverrideKeyLength = int.Parse(OutputsRowSize); mrc.Call().Trim(); } Shell("dspace swap \"" + outtablefn + "\" \"" + DfsOutputName + "\""); dfsclient.DeleteFile(outtablefn); } if (queryresults) { { int dtfieldindex = 0; foreach (DbColumn c in outputcols) { LogFieldInfo(dtfieldindex++, c, !joined); } } string sDfsOutputSize = dfsclient.GetFileSizeString(DfsOutputName); long DfsOutputSize = long.Parse(sDfsOutputSize); long NumRowsOutput = DfsOutputSize / OutputRowSize; if (0 != (DfsOutputSize % OutputRowSize)) { throw new Exception("Output file size miscalculation (DfsOutputSize{" + DfsOutputSize + "} % OutputRowSize{" + OutputRowSize + "}) for file: " + DfsOutputName); } long recordcount = NumRowsOutput; if (TopCount >= 0) { if (recordcount > TopCount) { recordcount = TopCount; if (topdfstemp) { string outtablefn = DfsOutputName + "_out_" + Guid.NewGuid().ToString(); { RemoteCall rc = GetRemoteCallWriteTop(); rc.OverrideInput = DfsOutputName + "@" + OutputsRowSize; rc.OverrideOutput = outtablefn + "@" + OutputsRowSize; rc.Call(TopCount.ToString()).Trim(); } Shell("dspace swap \"" + DfsOutputName + "\" \"" + outtablefn + "\""); dfsclient.DeleteFile(outtablefn); } } } DSpace_Log(" <recordcount>" + recordcount.ToString() + "</recordcount>"); DSpace_Log(" <recordsize>" + OutputsRowSize + "</recordsize>"); string sPartCount = dfsclient.GetFilePartCountString(DfsOutputName); DSpace_Log(" <parts>" + sPartCount + "</parts>"); DSpace_Log(" </queryresults>"); } else { DSpace_Log(shelloutputSelect1); string sTopOptions = "-"; if (joined) { sTopOptions += ";JOINED"; } string topoutput1; { RemoteCall rc = GetRemoteCallTop(DfsOutputName); rc.OverrideInput = DfsOutputName + "@" + OutputsRowSize; topoutput1 = rc.Call("\"" + TableName + "\" \"" + DfsOutputName + "\" \"" + Qa.QlArgsEscape(OutputRowInfo) + "\" \"" + OutputDisplayInfo + "\" " + TopCount.ToString() + " " + sTopOptions).Trim(); } DSpace_Log(topoutput1); { string delfile = DfsOutputName; { int iat = delfile.IndexOf('@'); if (-1 != iat) { delfile = delfile.Substring(0, iat); } } dfsclient.DeleteFile(delfile); } } }
protected override void Run() { string QlLeftTableName = DSpace_ExecArgs[0]; string LeftTableName = QlArgsUnescape(QlLeftTableName); string stype = DSpace_ExecArgs[1]; string QlRightTableName = DSpace_ExecArgs[2]; string RightTableName = QlArgsUnescape(QlRightTableName); string QlOn = DSpace_ExecArgs[3]; string On = QlArgsUnescape(QlOn); if (-1 != LeftTableName.IndexOf('\0')) { throw new NotSupportedException("Cannot JOIN with multiple left tables: " + LeftTableName); } if (-1 != RightTableName.IndexOf('\0')) { throw new NotSupportedException("Cannot JOIN with multiple right tables: " + RightTableName); } System.Xml.XmlDocument systables; using (GlobalCriticalSection.GetLock()) { systables = LoadSysTables_unlocked(); } System.Xml.XmlElement xeLeftTable = FindTable(systables, LeftTableName); if (null == xeLeftTable) { throw new Exception("Table (left) '" + LeftTableName + "' does not exist"); } string sLeftRowSize = xeLeftTable["size"].InnerText; int LeftRowSize = int.Parse(sLeftRowSize); string LeftDfsTableFilesInput = xeLeftTable["file"].InnerText + "@" + sLeftRowSize; if (LeftDfsTableFilesInput.StartsWith("qa://", true, null)) { throw new Exception("Cannot JOIN with system tables (left)"); } System.Xml.XmlElement xeRightTable = FindTable(systables, RightTableName); if (null == xeRightTable) { throw new Exception("Table (right) '" + RightTableName + "' does not exist"); } string sRightRowSize = xeRightTable["size"].InnerText; int RightRowSize = int.Parse(sRightRowSize); string RightDfsTableFilesInput = xeRightTable["file"].InnerText + "@" + sRightRowSize; if (RightDfsTableFilesInput.StartsWith("qa://", true, null)) { throw new Exception("Cannot JOIN with system tables (right)"); } string DfsTableFilesInput = LeftDfsTableFilesInput + ";" + RightDfsTableFilesInput; string DfsOutputName = "dfs://RDBMS_JoinOn_" + Guid.NewGuid().ToString() + Qa.DFS_TEMP_FILE_MARKER; int DfsOutputRowSize = LeftRowSize + RightRowSize; string LeftRowInfo; string LeftDisplayInfo; // Display List <DbColumn> LeftCols = new List <DbColumn>(); List <string> LeftColsWidths = new List <string>(); { StringBuilder sbRowInfo = new StringBuilder(); StringBuilder sbDisplayInfo = new StringBuilder(); // Display int totsize = 0; string xtablename = xeLeftTable["name"].InnerText; foreach (System.Xml.XmlNode xn in xeLeftTable.SelectNodes("column")) { if (0 != sbRowInfo.Length) { sbRowInfo.Append('\0'); sbDisplayInfo.Append(','); // Display } string stsize = xn["bytes"].InnerText; int tsize = int.Parse(stsize); string RealColName = xn["name"].InnerText; string UserColName = RealColName; string xcolname; if (-1 == UserColName.IndexOf('.')) { xcolname = xtablename + "." + UserColName; } else { xcolname = UserColName; } sbRowInfo.Append(xcolname); // Note: doesn't consider sub-select. sbRowInfo.Append('='); sbRowInfo.Append(stsize); sbDisplayInfo.Append(xn["type"].InnerText); // Display sbDisplayInfo.Append('='); // Display sbDisplayInfo.Append(xn["dw"].InnerText); // Display LeftColsWidths.Add(xn["dw"].InnerText); { DbColumn c; c.Type = DbType.Prepare(xn["type"].InnerText, tsize); c.RowOffset = totsize; c.ColumnName = xcolname; LeftCols.Add(c); } totsize += tsize; } LeftRowInfo = sbRowInfo.ToString(); LeftDisplayInfo = sbDisplayInfo.ToString(); // Display } string RightRowInfo; string RightDisplayInfo; // Display List <DbColumn> RightCols = new List <DbColumn>(); List <string> RightColsWidths = new List <string>(); { StringBuilder sbRowInfo = new StringBuilder(); StringBuilder sbDisplayInfo = new StringBuilder(); // Display int totsize = 0; string xtablename = xeRightTable["name"].InnerText; foreach (System.Xml.XmlNode xn in xeRightTable.SelectNodes("column")) { if (0 != sbRowInfo.Length) { sbRowInfo.Append('\0'); sbDisplayInfo.Append(','); // Display } string stsize = xn["bytes"].InnerText; int tsize = int.Parse(stsize); string RealColName = xn["name"].InnerText; string UserColName = RealColName; string xcolname; if (-1 == UserColName.IndexOf('.')) { xcolname = xtablename + "." + UserColName; } else { xcolname = UserColName; } sbRowInfo.Append(xcolname); // Note: doesn't consider sub-select. sbRowInfo.Append('='); sbRowInfo.Append(stsize); sbDisplayInfo.Append(xn["type"].InnerText); // Display sbDisplayInfo.Append('='); // Display sbDisplayInfo.Append(xn["dw"].InnerText); // Display RightColsWidths.Add(xn["dw"].InnerText); { DbColumn c; c.Type = DbType.Prepare(xn["type"].InnerText, tsize); c.RowOffset = totsize; c.ColumnName = xcolname; RightCols.Add(c); } totsize += tsize; } RightRowInfo = sbRowInfo.ToString(); RightDisplayInfo = sbDisplayInfo.ToString(); // Display } string on1, onop, on2; { string onargs = On; on1 = Qa.NextPart(ref onargs); onop = Qa.NextPart(ref onargs); on2 = Qa.NextPart(ref onargs); if (0 == on1.Length || 0 == onop.Length || 0 == on2.Length || 0 != onargs.Trim().Length) { throw new Exception("Invalid ON expression for JOIN: " + On); } } bool on1left; int on1colindex; { int on1colindexother; on1colindex = DbColumn.IndexOf(LeftCols, on1); on1colindexother = DbColumn.IndexOf(RightCols, on1); if (-1 != on1colindex) { if (-1 != on1colindexother) { throw new Exception("Column name " + on1 + " does not resolve to a single column (in left and right tables)"); } else { on1left = true; } } else { if (-1 != on1colindexother) { on1colindex = on1colindexother; on1left = false; } else { throw new Exception("No such column named " + on1); } } } bool on2left; int on2colindex; { int on2colindexother; on2colindex = DbColumn.IndexOf(LeftCols, on2); on2colindexother = DbColumn.IndexOf(RightCols, on2); if (-1 != on2colindex) { if (-1 != on2colindexother) { throw new Exception("Column name " + on2 + " does not resolve to a single column (in left and right tables)"); } else { on2left = true; } } else { if (-1 != on2colindexother) { on2colindex = on2colindexother; on2left = false; } else { throw new Exception("No such column named " + on2); } } } if ((on1left && on2left) || (!on1left && !on2left)) { string whicht; if (on1left) { whicht = "(left) " + LeftTableName; } else { whicht = "(right) " + RightTableName; } throw new Exception("ON expression is comparing columns from the same table: " + on1 + " and " + on2 + " are both part of " + whicht); } // Order ON columns: if (!on1left) { { string onx = on1; on1 = on2; on2 = onx; } { int onxcolindex = on1colindex; on1colindex = on2colindex; on2colindex = onxcolindex; } { on1left = true; on2left = false; } // Invert the operator.. switch (onop) { case "=": //onop = "="; break; case "!=": //onop = "!="; break; case "<": onop = ">"; break; case "<=": onop = ">="; break; case ">": onop = "<"; break; case ">=": onop = "<="; break; default: throw new Exception("Unhandled ON operation: " + onop); } On = on1 + " " + onop + " " + on2; QlOn = QlArgsEscape(On); } DbColumn on1col; if (on1left) { on1col = LeftCols[on1colindex]; } else { //on1col = RightCols[on1colindex]; throw new Exception("DEBUG: PrepareJoinOn: (!on1left)"); } if (on1col.Type.Size == 0 || on1col.Type.ID == DbTypeID.NULL) { throw new Exception("Invalid column for ON expression: " + on1); } DbColumn on2col; if (on2left) { //on2col = LeftCols[on2colindex]; throw new Exception("DEBUG: PrepareJoinOn: (on2left)"); } else { on2col = RightCols[on2colindex]; } if (on2col.Type.Size == 0 || on2col.Type.ID == DbTypeID.NULL) { throw new Exception("Invalid column for ON expression: " + on2); } int KeyLength = on1col.Type.Size; if (on2col.Type.Size > KeyLength) { KeyLength = on2col.Type.Size; } { MapReduceCall mrc = GetMapReduceCallJoinOn(DfsTableFilesInput); mrc.OverrideOutputMethod = "grouped"; mrc.OverrideInput = DfsTableFilesInput; mrc.OverrideOutput = DfsOutputName + "@" + DfsOutputRowSize; mrc.OverrideKeyLength = KeyLength; if (RDBMS_DBCORE.Qa.FaultTolerantExecution) { mrc.OverrideFaultTolerantExecutionMode = "enabled"; } mrc.Call("\"" + QlLeftTableName + "\" " + stype + " \"" + QlRightTableName + "\" \"" + QlOn + "\" \"" + on1col.RowOffset + "," + on1col.Type.Size + "=" + QlArgsEscape(on1col.Type.Name) + "\" \"" + on2col.RowOffset + "," + on2col.Type.Size + "=" + QlArgsEscape(on2col.Type.Name) + "\" \"" + DfsTableFilesInput + "\""); } { PrepareSelect.queryresults qr = new PrepareSelect.queryresults(); List <DbColumn> AllCols = new List <DbColumn>(LeftCols.Count + RightCols.Count); AllCols.AddRange(LeftCols); AllCols.AddRange(RightCols); qr.SetFields(AllCols); qr.temptable = DfsOutputName; qr.recordsize = LeftRowSize + RightRowSize; string sPartCount = Shell("dspace countparts \"" + DfsOutputName + "\"").Split('\n')[0].Trim(); qr.parts = int.Parse(sPartCount); string sDfsOutputSize = Shell("dspace filesize \"" + DfsOutputName + "\"").Split('\n')[0].Trim(); long DfsOutputSize = long.Parse(sDfsOutputSize); long NumRowsOutput = DfsOutputSize / qr.recordsize; qr.recordcount = NumRowsOutput; DSpace_Log(PrepareSelect.SetQueryResults(qr)); } }