ByteSlice Conv_String(ByteSlice value, int ResultSize, DbFunctionTools tools) { mstring x = tools.GetString(value); x = x.ToUpperM(); DbValue v = tools.AllocValue(x, ResultSize); return v.Eval(); }
public DateTime GetDateTime(ByteSlice input) { #if DEBUG_FTGET if (Types.IsNullValue(input)) { throw new Exception("DEBUG: GetDateTime: (Types.IsNullValue(input))"); } #endif List<byte> buf = AllocBuffer(input.Length - 1); ByteSlice.Prepare(input, 1, input.Length - 1).AppendTo(buf); Int64 x = Entry.BytesToLong(buf); FreeLastBuffer(buf); return new DateTime(x); }
void GetKeyPartForColumn(ByteSlice row, int ColIndex, List<byte> AppendKeyPart) { DbColumn ci = cols[ColIndex]; int StartOffset = ci.RowOffset; int Size = ci.Type.Size; ByteSlice cval = ByteSlice.Prepare(row, StartOffset, Size); if (ci.Type.Name.StartsWith("char")) { CellStringToCaseInsensitiveAppend(cval, AppendKeyPart); } else { cval.AppendTo(AppendKeyPart); } }
public int GetInt(ByteSlice input) { #if DEBUG_FTGET if (Types.IsNullValue(input)) { throw new Exception("DEBUG: GetInt: (Types.IsNullValue(input))"); } #endif List<byte> buf = AllocBuffer(input.Length - 1); ByteSlice.Prepare(input, 1, input.Length - 1).AppendTo(buf); Int32 x = Entry.BytesToInt(buf); FreeLastBuffer(buf); x = (Int32)Entry.ToUInt32(x); return x; }
public bool TestRow(ByteSlice row) { functools.ResetBuffers(); _CurRow = row; return this.eval.Test(); }
public static ByteSlice Create(ByteSlice data, int offset, int length) { return Prepare(data, offset, length); }
public void Add(ByteSlice key, ByteSlice value) { Add(key.buf, key.offset, key.length, value.buf, value.offset, value.length); }
public override void Map(ByteSlice row, MapOutput output) { if (JoinType.X == type) { ftools = new DbFunctionTools(); string QlLeftTableName = DSpace_ExecArgs[0]; LeftTableName = Qa.QlArgsUnescape(QlLeftTableName); string stype = DSpace_ExecArgs[1]; string QlRightTableName = DSpace_ExecArgs[2]; RightTableName = Qa.QlArgsUnescape(QlRightTableName); string QlOn = DSpace_ExecArgs[3]; On = Qa.QlArgsUnescape(QlOn); { string LeftColInfo = Qa.QlArgsUnescape(DSpace_ExecArgs[4]); int ileq = LeftColInfo.LastIndexOf('='); string sLeftType = LeftColInfo.Substring(ileq + 1); LeftType = DbType.Prepare(sLeftType); LeftConv = NeedConv(LeftType, RightType); string[] st = LeftColInfo.Substring(0, ileq).Split(','); LeftOffset = int.Parse(st[0]); LeftSize = int.Parse(st[1]); } { string RightColInfo = Qa.QlArgsUnescape(DSpace_ExecArgs[5]); int ileq = RightColInfo.LastIndexOf('='); string sRightType = RightColInfo.Substring(ileq + 1); RightType = DbType.Prepare(sRightType); RightConv = NeedConv(RightType, LeftType); string[] st = RightColInfo.Substring(0, ileq).Split(','); RightOffset = int.Parse(st[0]); RightSize = int.Parse(st[1]); } if (0 == string.Compare("INNER", stype, true)) { type = JoinType.INNER_JOIN; } else if (0 == string.Compare("LEFT_OUTER", stype, true)) { type = JoinType.LEFT_OUTER_JOIN; } else if (0 == string.Compare("RIGHT_OUTER", stype, true)) { type = JoinType.RIGHT_OUTER_JOIN; } else { throw new NotSupportedException("DEBUG: JOIN type not supported: " + stype); } string DfsTableFilesInput = DSpace_ExecArgs[6]; /* TableFileNames = DfsTableFilesInput.Split(';'); if (2 != TableFileNames.Length) { throw new Exception("DEBUG: Invalid number of tables"); } for (int it = 0; it < TableFileNames.Length; it++) { if (TableFileNames[it].StartsWith("dfs://", StringComparison.OrdinalIgnoreCase)) { TableFileNames[it] = TableFileNames[it].Substring(6); } int iat = TableFileNames[it].IndexOf('@'); if (-1 != iat) { TableFileNames[it] = TableFileNames[it].Substring(0, iat); } } * */ } int tableid = GetTableID(row); ByteSlice key; if (0 == tableid) { key = ByteSlice.Prepare(row, LeftOffset, LeftSize); if (null != LeftConv) { key = LeftConv(key, DSpace_KeyLength, ftools); } } else if (1 == tableid) { key = ByteSlice.Prepare(row, RightOffset, RightSize); if (null != RightConv) { key = RightConv(key, DSpace_KeyLength, ftools); } } else { throw new Exception("Map: Unexpected TableID: " + tableid); } List<byte> valuebuf = ftools.AllocBuffer(1 + 4 + row.Length); { DbValue tableiddbvalue = ftools.AllocValue(tableid); tableiddbvalue.Eval().AppendTo(valuebuf); } row.AppendTo(valuebuf); output.Add(key, ByteSlice.Prepare(valuebuf)); ftools.ResetBuffers(); }
public static ByteSlice Prepare(ByteSlice data, int offset, int length) { if (offset < 0 || length < 0 || offset + length > data.length) { IndexOutOfRangeException ior = new IndexOutOfRangeException("ByteSlice.Prepare(ByteSlice{offset=" + data.offset + ", length=" + data.length + "}, offset=" + offset + ", length=" + length + ")"); throw new ArgumentOutOfRangeException("Specified argument was out of the range of valid values. Index out of bounds: " + ior.Message, ior); // Preserve the old exception type. } ByteSlice result; result.buf = data.buf; result.offset = data.offset + offset; result.length = length; return result; }
public void PutByteSlice(ByteSlice bytes) { for (int i = 0; i < bytes.length; i++) { Buffer.Add(bytes[i]); } }
private static bool compareBytes(byte[] b1, ByteSlice b2) { if (b1.Length != b2.Length - 1) //include null front byte { return false; } for (int i = 0; i < b1.Length; i++) { if (b1[i] != b2[i + 1]) { return false; } } return true; }
int GetTableID(ByteSlice row) { return this.MapInputFileIndex; }
// Argument must start with Nullable byte. static void CellStringToCaseInsensitiveAppend(ByteSlice x, List<byte> append) { append.Add(x[0]); for (int i = 1; i + 2 <= x.Length; i += 2) { byte b0 = x[i + 0]; byte b1 = x[i + 1]; Types.UTF16BytesToLower(ref b0, ref b1); append.Add(b0); append.Add(b1); } }
ByteSlice Conv_LongToDouble(ByteSlice value, int ResultSize, DbFunctionTools tools) { long x = tools.GetLong(value); DbValue v = tools.AllocValue((double)x); return v.Eval(); }
public DbValue AllocValue(ByteSlice value, DbTypeID typeID) { return AllocValue(value, DbType.Prepare(value.Length, typeID)); }
ByteSlice Conv_IntToDouble(ByteSlice value, int ResultSize, DbFunctionTools tools) { int x = tools.GetInt(value); DbValue v = tools.AllocValue((double)x); return v.Eval(); }
public override void Reduce(ByteSlice key, IEnumerator<ByteSlice> values, MapReduceOutput output) { if (JoinType.X == type) { ftools = new DbFunctionTools(); string stype = DSpace_ExecArgs[1]; if (0 == string.Compare("INNER", stype, true)) { type = JoinType.INNER_JOIN; } else if (0 == string.Compare("LEFT_OUTER", stype, true)) { type = JoinType.LEFT_OUTER_JOIN; } else if (0 == string.Compare("RIGHT_OUTER", stype, true)) { type = JoinType.RIGHT_OUTER_JOIN; } else { throw new NotSupportedException("DEBUG: JOIN type not supported: " + stype); } t0 = new List<ByteSlice>(); t1 = new List<ByteSlice>(); } t0.Clear(); t1.Clear(); while (values.MoveNext()) { ByteSlice row = values.Current; int tableid = GetTableID(row); if (0 == tableid) { t0.Add(ByteSlice.Prepare(row, 5, row.Length - 5)); } else if (1 == tableid) { t1.Add(ByteSlice.Prepare(row, 5, row.Length - 5)); } else { throw new Exception("Reduce: Unexpected TableID: " + tableid); } } int t0Count = t0.Count; int t1Count = t1.Count; if (0 == t0Count) { if (JoinType.RIGHT_OUTER_JOIN == type) { for (int j = 0; j < t1Count; j++) { ByteSlice z = t1[j]; List<byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength); for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++) { valuebuf.Add(1); // IsNull=true. } z.AppendTo(valuebuf); output.Add(ByteSlice.Prepare(valuebuf)); } } } else if (0 == t1Count) { if (JoinType.LEFT_OUTER_JOIN == type) { for (int i = 0; i < t0Count; i++) { ByteSlice z = t0[i]; List<byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength); z.AppendTo(valuebuf); for (int n = 0; n < DSpace_OutputRecordLength - z.Length; n++) { valuebuf.Add(1); // IsNull=true. } output.Add(ByteSlice.Prepare(valuebuf)); } } } else { for (int i = 0; i < t0Count; i++) { for (int j = 0; j < t1Count; j++) { ByteSlice x = t0[i]; ByteSlice y = t1[j]; #if DEBUG if (DSpace_OutputRecordLength != x.Length + y.Length) { throw new Exception("DEBUG: JoinOn.Reduce: (DSpace_OutputRecordLength != x.Length + y.Length). x.Length=" + x.Length.ToString() + "; y.Length=" + y.Length.ToString() + ";DSpace_OutputRecordLength=" + DSpace_OutputRecordLength.ToString()); } #endif List<byte> valuebuf = ftools.AllocBuffer(DSpace_OutputRecordLength); x.AppendTo(valuebuf); y.AppendTo(valuebuf); output.Add(ByteSlice.Prepare(valuebuf)); } } } ftools.ResetBuffers(); }
int GetTableID(ByteSlice row) { return ftools.GetInt(ByteSlice.Prepare(row, 0, 1 + 4)); }
public static recordset PrepareRow(ByteSlice b) { recordset rs = recordset.Prepare(); rs.PutByteSlice(b); return rs; }
public override void Reduce(ByteSlice key, IEnumerator<ByteSlice> values, MapReduceOutput output) { output.Add(key); }
public static recordset Prepare(ByteSlice b) { recordset rs = recordset.Prepare(); for (int i = 0; i < b.Length ; i++) { rs.Buffer.Add(b[i]); } return rs; }
public ImmediateValue(IValueContext Context, ByteSlice value, DbType type) : base(Context) { this._value = value; this._type = type; }
public static DbRecordset Prepare(ByteSlice b) { DbRecordset dbrs; dbrs.rs = recordset.Prepare(b); return dbrs; }
public DbValue AllocValue(ByteSlice value, DbType type) { if (curval >= values.Count) { ImmediateValue val = new ImmediateValue(null, value, type); values.Add(val); curval++; return val; } else { ImmediateValue val = values[curval++]; val._value = value; val._type = type; return val; } }
public override void Map(ByteSlice line, MapOutput output) { output.Add(line, ByteSlice.Prepare()); }
public double GetDouble(ByteSlice input) { #if DEBUG_FTGET if (Types.IsNullValue(input)) { throw new Exception("DEBUG: GetDouble: (Types.IsNullValue(input))"); } #endif recordset rs = recordset.Prepare(ByteSlice.Prepare(input, 1, input.Length - 1)); double x = rs.GetDouble(); return x; }
public static bool IsNullValue(ByteSlice x) { #if DEBUG if (x.Length < 1 || (0 != x[0] && 1 != x[0])) { throw new Exception("DEBUG: IsNullValue: (x.Length < 1 || (0 != x[0] && 1 != x[0]))"); } #endif return 0 != x[0]; }
public override void Map(ByteSlice row, MapOutput output) { if (null == TableName) { InitFields(); } // Operate on values! ByteSlice key; bool keep = null == filter || filter.TestRow(row); if (DeleteFromFilter) { keep = !keep; } if (keep) { if (KeyOn != null) { orderbuf.Clear(); for (int iob = 0; iob < KeyOn.Count; iob++) { GetKeyPart gkp = KeyOn[iob]; gkp(row, orderbuf); } #if DEBUG string KeyOnStringValue = "<not evaluated yet>"; if(0 != ((orderbuf.Count - 1) % 2)) { KeyOnStringValue = "<not a string>"; } else { //KeyOnStringValue = System.Text.Encoding.Unicode.GetString(ByteSlice.Prepare(orderbuf, 1, orderbuf.Count - 1).ToBytes()); { System.Text.Encoding ue = new System.Text.UnicodeEncoding(false, false, true); // throwOnInvalidBytes=true try { KeyOnStringValue = ue.GetString(ByteSlice.Prepare(orderbuf, 1, orderbuf.Count - 1).ToBytes()); } catch { KeyOnStringValue = "<not a string>"; } } } #endif while (orderbuf.Count < DSpace_KeyLength) { orderbuf.Add(0); } key = ByteSlice.Prepare(orderbuf); } else // Use default key. { orderbuf.Clear(); Entry.ToBytesAppend(defkey, orderbuf); key = ByteSlice.Prepare(orderbuf); defkey = unchecked(defkey + DSpace_ProcessCount); } if (null != Updates) { updatebuf.Clear(); row.AppendTo(updatebuf); for (int ui = 0; ui < Updates.Count; ui++) { UpdateField uf = Updates[ui]; DbColumn ci = cols[uf.FieldIndex]; for (int i = 0; i < ci.Type.Size; i++) { updatebuf[ci.RowOffset + i] = uf.NewValue[i]; } } row = ByteSlice.Prepare(updatebuf); } } else { if (null == Updates) { if (null != ftools) { ftools.ResetBuffers(); } return; } if (KeyOn != null) { key = row; #if DEBUG throw new Exception("DEBUG: (!keep) && (KeyOn != null)"); #endif } else { // Use default key. orderbuf.Clear(); Entry.ToBytesAppend(defkey, orderbuf); key = ByteSlice.Prepare(orderbuf); defkey = unchecked(defkey + DSpace_ProcessCount); } } // If WhatFunctions, might need all input fields, so keep them here and filter out unwanted stuff in reduce. if (null != Whats && !WhatFunctions) { newfieldsbuf.Clear(); for (int iww = 0; iww < Whats.Count; iww++) { int wi = Whats[iww]; if (-1 == wi) { DbTypeID ltype; if (null == ftools) { ftools = new DbFunctionTools(); } List<byte> lbuf = ftools.AllocBuffer(); ByteSlice cval = Types.LiteralToValueBuffer(awhat[iww], out ltype, lbuf); int Size = cval.Length; #if DEBUG if(Size != OutputColumnSizes[iww]) { throw new Exception("DEBUG: " + awhat[iww] + ": (Size{" + Size + "} != OutputColumnSizes[iww]{" + OutputColumnSizes[iww] + "})"); } #endif cval.AppendTo(newfieldsbuf); } else { DbColumn ci = cols[wi]; int StartOffset = ci.RowOffset; int Size = ci.Type.Size; #if DEBUG if(Size != OutputColumnSizes[iww]) { throw new Exception("DEBUG: (Size != OutputColumnSizes[iww])"); } #endif ByteSlice cval = ByteSlice.Prepare(row, StartOffset, Size); cval.AppendTo(newfieldsbuf); } } row = ByteSlice.Prepare(newfieldsbuf); } output.Add(key, row); if (null != ftools) { ftools.ResetBuffers(); } }
public void SetValue(ByteSlice value) { this._value = value; this._type = DbType.Prepare(value.Length, _type.ID); }
public override void Reduce(ByteSlice key, IEnumerator<ByteSlice> values, MapReduceOutput output) { #if DEBUG string KeyStringValue = "<not evaluated yet>"; if(0 != ((key.Length - 1) % 2)) { KeyStringValue = "<not a string>"; } else { //KeyStringValue = System.Text.Encoding.Unicode.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes()); { System.Text.Encoding ue = new System.Text.UnicodeEncoding(false, false, true); // throwOnInvalidBytes=true try { KeyStringValue = ue.GetString(ByteSlice.Prepare(key, 1, key.Length - 1).ToBytes()); } catch { KeyStringValue = "<not a string>"; } } } #endif if (null == sOptions) { // Done early to detect SFUNC sOptions = (DSpace_ExecArgs.Length > 8) ? DSpace_ExecArgs[8] : "-"; WhatFunctions = -1 != sOptions.IndexOf("SFUNC"); GroupBy = -1 != sOptions.IndexOf("GBY"); } if (!WhatFunctions) { // Normal: no functions (aggregates or scalars). if (GroupBy) { values.Reset(); while (values.MoveNext()) { output.Add(values.Current); break; } } else { values.Reset(); while (values.MoveNext()) { output.Add(values.Current); } } } else //if(WhatFunctions) { if (null == TableName) { InitFields(); // Note: only called if functions (aggregates or scalars)! agtools = new DbFunctionTools(); } bool GettingFieldTypeStrings = null == FieldTypeStrings; if (GettingFieldTypeStrings) { FieldTypeStrings = new List<string>(); } rows.Clear(); outvalues.Clear(); values.Reset(); while (values.MoveNext()) { ByteSlice row = values.Current; rows.Add(row); outvalues.Add(agtools.AllocBuffer(row.Length > 256 ? Entry.Round2Power(row.Length) : 256)); } //List<SelectClause> sclauses bool NewSelectClauses = null == sclauses; if (NewSelectClauses) { sclauses = new List<SelectClause>(Whats.Count); } for (int iww = 0; iww < Whats.Count; iww++) { int wi = Whats[iww]; string Source = awhat[iww]; string calls = null; if (NewSelectClauses) { sclauses.Add(new SelectClause(agtools, cols)); calls = Source; } SelectClause sclause = sclauses[iww]; List<DbValue> results = sclause.ProcessSelectPart(calls, rows); #if DEBUG if(1 != results.Count && results.Count != outvalues.Count) { throw new Exception("DEBUG: (WhatFunctions) && (1 != results.Count && results.Count != outvalues.Count)"); } #endif int outvaluesCount = outvalues.Count; DbType rtype = DbType.PrepareNull(); // Just for init. bool JustOneResult = 1 == results.Count; bool DifferentSize = false; for (int ir = 0; ir < outvaluesCount; ir++) { ByteSlice bs; int ix = ir; if (JustOneResult) { ix = 0; } bs = results[ix].Eval(out rtype); bs.AppendTo(outvalues[ir]); for (int vdiff = OutputColumnSizes[iww] - bs.Length; vdiff > 0; vdiff--) { DifferentSize = true; outvalues[ir].Add(0); } } if (GettingFieldTypeStrings) { if (DifferentSize) { rtype = DbType.Prepare(OutputColumnSizes[iww], rtype.ID); } FieldTypeStrings.Add(rtype.Name); } } { int outvaluesCount = outvalues.Count; for (int iv = 0; iv < outvaluesCount; iv++) { output.Add(ByteSlice.Prepare(outvalues[iv])); if (GroupBy) { break; } } } //if(null != agtools) { agtools.ResetBuffers(); // Last. Important! } } }