public bool TestRow(ByteSlice row)
{
functools.ResetBuffers();
_CurRow = row;
return(this.eval.Test());
}
public override void Reduce(ByteSlice key, IEnumerator <ByteSlice> values, MapReduceOutput output)
{
#if DEBUG
string KeyStringValue = "<not evaluated yet>";
if (0 != ((key.Length - 1) % 2))
{
KeyStringValue = "<not a string>";
}
else
{
//KeyStringValue = System.Text.Encoding.Unicode.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes());
{
System.Text.Encoding ue = new System.Text.UnicodeEncoding(false, false, true); // throwOnInvalidBytes=true
try
{
KeyStringValue = ue.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes());
}
catch
{
KeyStringValue = "<not a string>";
}
}
}
#endif
if (null == sOptions)
{
// Done early to detect SFUNC
sOptions = (DSpace_ExecArgs.Length > 8) ? DSpace_ExecArgs[8] : "-";
WhatFunctions = -1 != sOptions.IndexOf("SFUNC");
GroupBy = -1 != sOptions.IndexOf("GBY");
}
if (!WhatFunctions)
{
// Normal: no functions (aggregates or scalars).
if (GroupBy)
{
values.Reset();
while (values.MoveNext())
{
output.Add(values.Current);
break;
}
}
else
{
values.Reset();
while (values.MoveNext())
{
output.Add(values.Current);
}
}
}
else //if(WhatFunctions)
{
if (null == TableName)
{
InitFields(); // Note: only called if functions (aggregates or scalars)!
agtools = new DbFunctionTools();
}
bool GettingFieldTypeStrings = null == FieldTypeStrings;
if (GettingFieldTypeStrings)
{
FieldTypeStrings = new List <string>();
}
rows.Clear();
outvalues.Clear();
values.Reset();
while (values.MoveNext())
{
ByteSlice row = values.Current;
rows.Add(row);
outvalues.Add(agtools.AllocBuffer(row.Length > 256 ? Entry.Round2Power(row.Length) : 256));
}
//List<SelectClause> sclauses
bool NewSelectClauses = null == sclauses;
if (NewSelectClauses)
{
sclauses = new List <SelectClause>(Whats.Count);
}
for (int iww = 0; iww < Whats.Count; iww++)
{
int wi = Whats[iww];
string Source = awhat[iww];
string calls = null;
if (NewSelectClauses)
{
sclauses.Add(new SelectClause(agtools, cols));
calls = Source;
}
SelectClause sclause = sclauses[iww];
List <DbValue> results = sclause.ProcessSelectPart(calls, rows);
#if DEBUG
if (1 != results.Count && results.Count != outvalues.Count)
{
throw new Exception("DEBUG: (WhatFunctions) && (1 != results.Count && results.Count != outvalues.Count)");
}
#endif
int outvaluesCount = outvalues.Count;
DbType rtype = DbType.PrepareNull(); // Just for init.
bool JustOneResult = 1 == results.Count;
bool DifferentSize = false;
for (int ir = 0; ir < outvaluesCount; ir++)
{
ByteSlice bs;
int ix = ir;
if (JustOneResult)
{
ix = 0;
}
bs = results[ix].Eval(out rtype);
bs.AppendTo(outvalues[ir]);
for (int vdiff = OutputColumnSizes[iww] - bs.Length; vdiff > 0; vdiff--)
{
DifferentSize = true;
outvalues[ir].Add(0);
}
}
if (GettingFieldTypeStrings)
{
if (DifferentSize)
{
rtype = DbType.Prepare(OutputColumnSizes[iww], rtype.ID);
}
FieldTypeStrings.Add(rtype.Name);
}
}
{
int outvaluesCount = outvalues.Count;
for (int iv = 0; iv < outvaluesCount; iv++)
{
output.Add(ByteSlice.Prepare(outvalues[iv]));
if (GroupBy)
{
break;
}
}
}
//if(null != agtools)
{
agtools.ResetBuffers(); // Last. Important!
}
}
}
public override void Reduce(ByteSlice key, IEnumerator <ByteSlice> values, MapReduceOutput output)
{
if (JoinType.X == type)
{
ftools = new DbFunctionTools();
string stype = DSpace_ExecArgs[1];
if (0 == string.Compare("INNER", stype, true))
{
type = JoinType.INNER_JOIN;
}
else if (0 == string.Compare("LEFT_OUTER", stype, true))
{
type = JoinType.LEFT_OUTER_JOIN;
}
else if (0 == string.Compare("RIGHT_OUTER", stype, true))
{
type = JoinType.RIGHT_OUTER_JOIN;
}
else
{
throw new NotSupportedException("DEBUG: JOIN type not supported: " + stype);
}
t0 = new List <ByteSlice>();
t1 = new List <ByteSlice>();
}
t0.Clear();
t1.Clear();
while (values.MoveNext())
{
ByteSlice row = values.Current;
int tableid = GetTableID(row);
if (0 == tableid)
{
t0.Add(ByteSlice.Prepare(row, 5, row.Length - 5));
}
else if (1 == tableid)
{
t1.Add(ByteSlice.Prepare(row, 5, row.Length - 5));
}
else
{
throw new Exception("Reduce: Unexpected TableID: " + tableid);
}
}
int t0Count = t0.Count;
int t1Count = t1.Count;
if (0 == t0Count)
{
if (JoinType.RIGHT_OUTER_JOIN == type)
{
for (int j = 0; j < t1Count; j++)
{
ByteSlice z = t1[j];
List <byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++)
{
valuebuf.Add(1); // IsNull=true.
}
z.AppendTo(valuebuf);
output.Add(ByteSlice.Prepare(valuebuf));
}
}
}
else if (0 == t1Count)
{
if (JoinType.LEFT_OUTER_JOIN == type)
{
for (int i = 0; i < t0Count; i++)
{
ByteSlice z = t0[i];
List <byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
z.AppendTo(valuebuf);
for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++)
{
valuebuf.Add(1); // IsNull=true.
}
output.Add(ByteSlice.Prepare(valuebuf));
}
}
}
else
{
for (int i = 0; i < t0Count; i++)
{
for (int j = 0; j < t1Count; j++)
{
ByteSlice x = t0[i];
ByteSlice y = t1[j];
#if DEBUG
if (DSpace_OutputRecordLength != x.Length + y.Length)
{
throw new Exception("DEBUG: JoinOn.Reduce: (DSpace_OutputRecordLength != x.Length + y.Length). x.Length="
+ x.Length.ToString() + "; y.Length=" + y.Length.ToString() + ";DSpace_OutputRecordLength="
+ DSpace_OutputRecordLength.ToString());
}
#endif
List <byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength);
x.AppendTo(valuebuf);
y.AppendTo(valuebuf);
output.Add(ByteSlice.Prepare(valuebuf));
}
}
}
ftools.ResetBuffers();
}
public override void Map(ByteSlice row, MapOutput output)
{
if (null == TableName)
{
InitFields();
}
// Operate on values!
ByteSlice key;
bool keep = null == filter || filter.TestRow(row);
if (DeleteFromFilter)
{
keep = !keep;
}
if (keep)
{
if (KeyOn != null)
{
orderbuf.Clear();
for (int iob = 0; iob < KeyOn.Count; iob++)
{
GetKeyPart gkp = KeyOn[iob];
gkp(row, orderbuf);
}
#if DEBUG
string KeyOnStringValue = "<not evaluated yet>";
if (0 != ((orderbuf.Count - 1) % 2))
{
KeyOnStringValue = "<not a string>";
}
else
{
//KeyOnStringValue = System.Text.Encoding.Unicode.GetString(ByteSlice.Prepare(orderbuf, 1, orderbuf.Count - 1).ToBytes());
{
System.Text.Encoding ue = new System.Text.UnicodeEncoding(false, false, true); // throwOnInvalidBytes=true
try
{
KeyOnStringValue = ue.GetString(ByteSlice.Prepare(orderbuf, 1, orderbuf.Count - 1).ToBytes());
}
catch
{
KeyOnStringValue = "<not a string>";
}
}
}
#endif
while (orderbuf.Count < DSpace_KeyLength)
{
orderbuf.Add(0);
}
key = ByteSlice.Prepare(orderbuf);
}
else // Use default key.
{
orderbuf.Clear();
Entry.ToBytesAppend(defkey, orderbuf);
key = ByteSlice.Prepare(orderbuf);
defkey = unchecked (defkey + DSpace_ProcessCount);
}
if (null != Updates)
{
updatebuf.Clear();
row.AppendTo(updatebuf);
for (int ui = 0; ui < Updates.Count; ui++)
{
UpdateField uf = Updates[ui];
DbColumn ci = cols[uf.FieldIndex];
for (int i = 0; i < ci.Type.Size; i++)
{
updatebuf[ci.RowOffset + i] = uf.NewValue[i];
}
}
row = ByteSlice.Prepare(updatebuf);
}
}
else
{
if (null == Updates)
{
if (null != ftools)
{
ftools.ResetBuffers();
}
return;
}
if (KeyOn != null)
{
key = row;
#if DEBUG
throw new Exception("DEBUG: (!keep) && (KeyOn != null)");
#endif
}
else
{
// Use default key.
orderbuf.Clear();
Entry.ToBytesAppend(defkey, orderbuf);
key = ByteSlice.Prepare(orderbuf);
defkey = unchecked (defkey + DSpace_ProcessCount);
}
}
// If WhatFunctions, might need all input fields, so keep them here and filter out unwanted stuff in reduce.
if (null != Whats && !WhatFunctions)
{
newfieldsbuf.Clear();
for (int iww = 0; iww < Whats.Count; iww++)
{
int wi = Whats[iww];
if (-1 == wi)
{
DbTypeID ltype;
if (null == ftools)
{
ftools = new DbFunctionTools();
}
List <byte> lbuf = ftools.AllocBuffer();
ByteSlice cval = Types.LiteralToValueBuffer(awhat[iww], out ltype, lbuf);
int Size = cval.Length;
#if DEBUG
if (Size != OutputColumnSizes[iww])
{
throw new Exception("DEBUG: " + awhat[iww] + ": (Size{" + Size + "} != OutputColumnSizes[iww]{" + OutputColumnSizes[iww] + "})");
}
#endif
cval.AppendTo(newfieldsbuf);
}
else
{
DbColumn ci = cols[wi];
int StartOffset = ci.RowOffset;
int Size = ci.Type.Size;
#if DEBUG
if (Size != OutputColumnSizes[iww])
{
throw new Exception("DEBUG: (Size != OutputColumnSizes[iww])");
}
#endif
ByteSlice cval = ByteSlice.Prepare(row, StartOffset, Size);
cval.AppendTo(newfieldsbuf);
}
}
row = ByteSlice.Prepare(newfieldsbuf);
}
output.Add(key, row);
if (null != ftools)
{
ftools.ResetBuffers();
}
}
public override void Map(ByteSlice row, MapOutput output)
{
if (JoinType.X == type)
{
ftools = new DbFunctionTools();
string QlLeftTableName = DSpace_ExecArgs[0];
LeftTableName = Qa.QlArgsUnescape(QlLeftTableName);
string stype = DSpace_ExecArgs[1];
string QlRightTableName = DSpace_ExecArgs[2];
RightTableName = Qa.QlArgsUnescape(QlRightTableName);
string QlOn = DSpace_ExecArgs[3];
On = Qa.QlArgsUnescape(QlOn);
{
string LeftColInfo = Qa.QlArgsUnescape(DSpace_ExecArgs[4]);
int ileq = LeftColInfo.LastIndexOf('=');
string sLeftType = LeftColInfo.Substring(ileq + 1);
LeftType = DbType.Prepare(sLeftType);
LeftConv = NeedConv(LeftType, RightType);
string[] st = LeftColInfo.Substring(0, ileq).Split(',');
LeftOffset = int.Parse(st[0]);
LeftSize = int.Parse(st[1]);
}
{
string RightColInfo = Qa.QlArgsUnescape(DSpace_ExecArgs[5]);
int ileq = RightColInfo.LastIndexOf('=');
string sRightType = RightColInfo.Substring(ileq + 1);
RightType = DbType.Prepare(sRightType);
RightConv = NeedConv(RightType, LeftType);
string[] st = RightColInfo.Substring(0, ileq).Split(',');
RightOffset = int.Parse(st[0]);
RightSize = int.Parse(st[1]);
}
if (0 == string.Compare("INNER", stype, true))
{
type = JoinType.INNER_JOIN;
}
else if (0 == string.Compare("LEFT_OUTER", stype, true))
{
type = JoinType.LEFT_OUTER_JOIN;
}
else if (0 == string.Compare("RIGHT_OUTER", stype, true))
{
type = JoinType.RIGHT_OUTER_JOIN;
}
else
{
throw new NotSupportedException("DEBUG: JOIN type not supported: " + stype);
}
string DfsTableFilesInput = DSpace_ExecArgs[6];
/*
* TableFileNames = DfsTableFilesInput.Split(';');
* if (2 != TableFileNames.Length)
* {
* throw new Exception("DEBUG: Invalid number of tables");
* }
* for (int it = 0; it < TableFileNames.Length; it++)
* {
* if (TableFileNames[it].StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
* {
* TableFileNames[it] = TableFileNames[it].Substring(6);
* }
* int iat = TableFileNames[it].IndexOf('@');
* if (-1 != iat)
* {
* TableFileNames[it] = TableFileNames[it].Substring(0, iat);
* }
* }
* */
}
int tableid = GetTableID(row);
ByteSlice key;
if (0 == tableid)
{
key = ByteSlice.Prepare(row, LeftOffset, LeftSize);
if (null != LeftConv)
{
key = LeftConv(key, DSpace_KeyLength, ftools);
}
}
else if (1 == tableid)
{
key = ByteSlice.Prepare(row, RightOffset, RightSize);
if (null != RightConv)
{
key = RightConv(key, DSpace_KeyLength, ftools);
}
}
else
{
throw new Exception("Map: Unexpected TableID: " + tableid);
}
List <byte> valuebuf = ftools.AllocBuffer(1 + 4 + row.Length);
{
DbValue tableiddbvalue = ftools.AllocValue(tableid);
tableiddbvalue.Eval().AppendTo(valuebuf);
}
row.AppendTo(valuebuf);
output.Add(key, ByteSlice.Prepare(valuebuf));
ftools.ResetBuffers();
}