/// <summary>
/// Copy constructor for a data item
/// </summary>
/// <param name="di">The data item to copy</param>
/// <param name="c">The number of attributes more to track than the input data item has</param>
/// <param name="r">Delegate to release data item back to data pool</param>
public DataItem(DataItem di, uint c, Utility.DataItemPool.ReleaseDataItem r)
{
cAttrs = (int)c + di.Count;
fEOF = di.fEOF;
TimeStamp = di.TimeStamp;
reldataitem = r;
alValues = dvm.GetALValues();
foreach (object o in di.alValues)
{
AddValue(o);
}
}
public DataItem PreProcess(DataItem di)
{
//<Time, VID, Spd, Xway, Lane, Dir, Seg, Pos>
if ((int)di[1] == -1) //we need to turn it into a proper punctuation
{
const int WildCards = 7;
Punctuation p = new Punctuation(WildCards + 1);
p.AddValue(new Punctuation.LiteralPattern(di[0]));
for (int i = 0; i < WildCards; i++)
p.AddValue(new Punctuation.WildcardPattern());
return p;
}
else
return di;
//<Time, VID, Spd, Xway, Lane, Dir, Seg, Pos>
}
/// <summary>
/// Trivial step functionality : ignore the input continue
/// </summary>
/// <param name="di">The input data item</param>
/// <param name="ldi">The output data items</param>
public static void StepTrivial(DataItem di, List<DataItem>ldi)
{
}
private void NewGroup(DataItem di, List<Group> al, out Group g)
{
g = new Group();
g.keys = new object[attrs.Length];
int iKey = 0;
foreach (int a in attrs)
g.keys[iKey++] = di[a];
if (al == null)
al = new List<Group>();
g.hashcode = di.GetSpecificHashCode(attrs);
al.Add(g);
ht[g.hashcode] = al;
}
/// <summary>
/// Find the appropriate group for this DataItem object
/// </summary>
/// <param name="di">The input DataItem</param>
/// <param name="g">The group that the DataItem belongs to</param>
/// <returns>True if this group already exists, false if this is a new group</returns>
protected bool FindGroup(DataItem di, out Group g)
{
bool fExists = false;
if (attrs == null)
{
fExists = (gTotal != null);
if (!fExists)
{
gTotal = new Group();
gTotal.keys = null;
}
g = gTotal;
return fExists;
}
int hc = di.GetSpecificHashCode(attrs);
if (!ht.ContainsKey(hc))
NewGroup(di, null, out g);
else
{
bool fMatch = false;
bool fEq;
List<Group> al = ht[hc];
g = null;
for (int i = 0; fMatch == false && i < al.Count; i++)
{
g = al[i];
fEq = true;
for (int j = 0; fEq && j < g.keys.Length; j++)
fEq &= di[attrs[j]].Equals(g.keys[j]);
fMatch = fEq;
}
fExists = fMatch;
if (fMatch == false)
NewGroup(di, al, out g);
}
return fExists;
}
private void SStep(DataItem di, List<DataItem> ldi)
{
al.Add(di);
}
/// <summary> Read data from the stream as quickly as possible, and store it in a queue </summary>
protected void StrReader()
{
const int STREAMDATABUFFER = 100;
//DataItem di = null;
bool eof = false;
BinaryFormatter bf = new BinaryFormatter();
DataItem[] rgdi = new DataItem[STREAMDATABUFFER];
int iData = 0;
do
{
while (iData < STREAMDATABUFFER && nsClient.DataAvailable && !eof)
{
try
{
rgdi[iData] = (DataItem)bf.Deserialize(nsClient);
eof = rgdi[iData].EOF;
}
catch (System.IO.IOException)
{
//Assume that the connection was closed (though I have no way of knowing...)
eof = true;
}
catch (System.Exception ex)
{
Console.WriteLine("Something bad happened when we tried to read from the stream: {0} ({1})",
ex.Message, ((ex.InnerException != null) ? ex.InnerException.Message : ""));
eof = true;
}
iData++;
}
lock (queBuffer)
{
for (int i = 0; i < iData; i++)
{
queBuffer.Enqueue(rgdi[i]);
rgdi[i] = null;
}
iData = 0;
}
} while (!eof);
nsClient.Close();
if (rgdi.Length == 0 || (rgdi[rgdi.Length-1] != null && !rgdi[rgdi.Length-1].EOF))
{
lock (queBuffer)
{
DataItem di = new DataItem(2, null);
di.EOF = true;
queBuffer.Enqueue(di);
}
}
}
internal void SelStep(DataItem di, List<DataItem> ldi)
{
if (expr.Evaluate(di))
ldi.Add(di);
}
/// <summary>
/// Returns whether or not this punctuation matches the given data item
/// </summary>
/// <param name="d">The data item to match on</param>
/// <returns>true if the punctuation matches the data item</returns>
public override bool Match(DataItem d)
{
bool fMatch = false;
for (int i = 0; i < listP.Count && fMatch == false; i++)
fMatch = listP[i].Match(d);
return fMatch;
}
public DataItem Type3MapConvert(DataItem di)
{
//<Time, VID, Xway, QID, Day, Bal>
di.AddCapacity(1);
di.AddValue(Convert.ToInt32(di[0]) + (int)(DateTime.Now.Subtract(di.TimeStamp).TotalSeconds));
return di;
//<Time, VID, Xway, QID, Day, Bal, Emit>
}
public DataItem Type3Relation(DataItem di, RelationTable rt)
{
//<Time, VID, Xway, QID, Day, InTime(DateTime)>
di.AddCapacity(1);
object[] keys = { di[1] };
lock (rt)
{
DataRow drow = rt.Find(keys);
if (drow == null)
{
drow = rt.GetRow();
drow["VID"] = Convert.ToInt32(di[1]);
drow["Bal"] = 0;
drow["Time"] = DateTime.Now.Subtract(di.TimeStamp).TotalSeconds;
rt.Add(drow);
//Console.WriteLine("We had to make a new balance for a type 3 query");
}
di.AddValue(drow["Bal"]);
}
return di;
//<Time, VID, Xway, QID, Day, Bal>
}
public DataItem Type2Relation(DataItem di, RelationTable rt)
{
//<Time, VID, QID>
if (!(di is Punctuation))
{
object[] keys = { di[1] }; //VID
int bal, time;
di.AddCapacity(2);
lock (rt)
{
DataRow drow = rt.Find(keys);
if (drow == null)
{
drow = rt.GetRow();
drow["VID"] = Convert.ToInt32(di[1]);
drow["Bal"] = 0;
drow["Time"] = DateTime.Now.Subtract(di.TimeStamp).TotalSeconds;
rt.Add(drow);
//Console.WriteLine("We had to make a new balance for a type 2 query");
}
bal = drow["Bal"];
time = drow["Time"];
}
di.AddValue(bal);
di.AddValue(time);
return di;
}
else
return di;
//<Time, VID, QID, Bal, UTime>
}
public DataItem SStatsLAVInsertion(DataItem di, RelationTable rt)
{
if (!(di is Punctuation))
{
object[] keys = { di[2], di[0], di[1] }; //Seg, Xway, Dir
lock (rt)
{
DataRow drow = rt.Find(keys);
if (drow != null)
{
drow["LAV"] = di[4];
drow["LAVWID"] = di[3];
}
else
{
drow = rt.GetRow();
drow["Seg"] = di[2];
drow["Xway"] = di[0];
drow["Dir"] = di[1];
drow["LAV"] = di[4];
drow["LAVWID"] = di[3];
drow["Count"] = -1; //we have no idea what it might be
drow["CountWID"] = di[3];
rt.Add(drow);
}
}
}
else //we need to check every segment
{
Punctuation p = di as Punctuation;
ulong WID = (ulong)((Punctuation.LiteralPattern)p[3]).Value;
for (int s = 0; s < SEGMENTS; s++)
{
for (int x = 0; x < EXPRESSWAYS; x++)
{
for (int d = 0; d < DIRECTIONS; d++)
{
object[] keys = { s, x, d };
lock (rt)
{
DataRow drow = rt.Find(keys);
if (drow != null)
{
if ((ulong)drow["LAVWID"] < WID)
{
drow["LAV"] = -1;
drow["LAVWID"] = WID;
}
}
else
{
drow = rt.GetRow();
drow["Seg"] = s;
drow["Xway"] = x;
drow["Dir"] = d;
drow["LAV"] = -1;
drow["LAVWID"] = 0;
drow["Count"] = -1;
drow["CountWID"] = 0;
rt.Add(drow);
}
}
}
}
}
}
return di;
}
public DataItem QuoteTolls(DataItem di, RelationTable rt)
{
//<Time, VID, Xway, Lane, Dir, Seg, ?isNSS, ?isAIR, LAV, Count>
if (!(di is Punctuation))
{
uint toll,
LAV = Convert.ToUInt32(di[8]),
Count = Convert.ToUInt32(di[9]);
int isAccident = Convert.ToInt32(di[7]);
if (isAccident != -1 || Count <= 50 || LAV >= 40)
toll = 0;
else
{
toll = 2 * (uint)Math.Pow((Count - 50), 2.0);
//Console.WriteLine("Toll Quote: {0}, Count: {1}, LAV: {2}", toll, Count, LAV);
lock (rt)
{
DataRow drow = rt.GetRow();
drow["VID"] = di[1];
drow["Seg"] = di[5];
drow["LAV"] = LAV;
drow["Toll"] = toll;
rt.Add(drow);
}
}
}
//<Time, VID, Xway, Lane, Dir, Seg, ?isNSS, ?isAIR, LAV, Count>
return di;
}
/// <summary> Add a data item to the queue </summary>
/// <param name="d">Data to enqueue</param>
public void Push(DataItem d)
{
if (d != null)
{
lock (data)
{
data.Enqueue(d);
if (d is Punctuation)
cPunct++;
hasEOF |= d.EOF;
}
}
}
/// <summary>
/// Returns whether or not this punctuation matches the given data item
/// </summary>
/// <param name="d">The data item to match on</param>
/// <returns>true if the punctuation matches the data item</returns>
public virtual bool Match(DataItem d)
{
bool m = true;
for (int i = 0; m == true && i < alValues.Count; i++)
{
Pattern p = alValues[i] as Pattern;
m &= p.Match(d[i]);
}
return m;
}
internal void DEStep(DataItem di, List<DataItem> ldi)
{
bool fNew = false;
int hc = di.GetHashCode();
if (ht.ContainsKey(hc) == false)
{
List<DataItem> al = new List<DataItem>();
al.Add(di);
ht[hc] = al;
fNew = true;
}
else
{
List<DataItem> al = ht[hc];
if (al.IndexOf(di) == -1)
{
al.Add(di);
ht[hc] = al;
fNew = true;
}
}
if (fNew)
ldi.Add(di);
}
internal void GBAVGStep(DataItem di, List<DataItem> ldi)
{
Group g;
if (FindGroup(di, out g))
{
((State)g.oState).count = ((State)g.oState).count + 1;
((State)g.oState).sum = ((State)g.oState).sum + (double)di[val];
}
else
{
g.oState = new State();
((State)g.oState).count = 1;
((State)g.oState).sum = (double)di[val];
}
g.oResult = ((State)g.oState).sum / ((State)g.oState).count;
}
internal void SelStep(DataItem di, List<DataItem> ldi)
{
if (expr.Evaluate(di) && rndDrop.Next(freq) != 0)
ldi.Add(di);
else
//ReleaseDataItem(di);
di.Dispose();
}
internal void GBCOUNTStep(DataItem di, List<DataItem> ldi)
{
Group g;
if (FindGroup(di, out g))
g.oResult = ((int)g.oResult) + 1;
else
g.oResult = 1;
}
/// <summary> Read data from the stream as quickly as possible, and store it in a queue </summary>
/// <remarks> Will recieve the header file before it tries to read raw data</remarks>
/// <remarks> Todo: Support for eof/stream termination, Non-blocking read of data items</remarks>
protected void StrReader()
{
const int STREAMDATABUFFER = 100;
bool eof = false,
receivedHeader = false;
BinaryFormatter bf = new BinaryFormatter();
DataItem[] rgdi = new DataItem[STREAMDATABUFFER];
int iData = 0,
fiter = 0; //format iterator
while (!receivedHeader) //try to get the initial setup for data input
{
if (nsClient.DataAvailable)
{
try
{
int inByte = nsClient.ReadByte();
while ((char)inByte != '#' && inByte != -1) //# indicates the termination of the header
{
format += (char)inByte;
inByte = nsClient.ReadByte();
}
if ((char)inByte == '#')
receivedHeader = true;
}
catch (System.Exception ex)
{
Console.WriteLine("Something bad happened when we tried to read the header: {0} ({1})",
ex.Message, ((ex.InnerException != null) ? ex.InnerException.Message : ""));
eof = true;
}
}
}
time.Start();
do
{
while (iData < STREAMDATABUFFER && nsClient.DataAvailable && !eof)
{
try
{
rgdi[iData] = new DataItem(format.Length, null);
while (fiter < format.Length && !eof) //TODO: this will lock out if an item is not completed, we may still want to push previously completed items
{
if (nsClient.DataAvailable)
{
switch (format[fiter]) //get the raw input based on the type
{
case 'i': //the input is a 32 bit int
{
byte[] inbytes = new byte[4];
nsClient.Read(inbytes, 0, 4); //read in the bytes
rgdi[iData].AddValue(BitConverter.ToInt32(inbytes, 0));
break;
}
case 'l': //the input is a 64 bit long (int)
{
byte[] inbytes = new byte[8];
nsClient.Read(inbytes, 0, 8); //read in the bytes
rgdi[iData].AddValue(BitConverter.ToInt64(inbytes, 0));
break;
}
case 'c': //the input is a char
{
rgdi[iData].AddValue((char)nsClient.ReadByte());
break;
}
case 'b': //the input is a bool
{
byte[] inbytes = new byte[1];
nsClient.Read(inbytes, 0, 1);
rgdi[iData].AddValue(BitConverter.ToBoolean(inbytes, 0));
break;
}
default:
{
throw new System.Exception("The specified type from the header could not be identified");
}
}
eof = rgdi[iData].EOF; //this won't work
fiter++;
}
}
}
catch (System.IO.IOException)
{
//Assume that the connection was closed (though I have no way of knowing...)
eof = true;
}
catch (System.Exception ex)
{
Console.WriteLine("Something bad happened when we tried to read from the stream: {0} ({1})",
ex.Message, ((ex.InnerException != null) ? ex.InnerException.Message : ""));
eof = true;
}
iData++;
fiter = 0;
}
lock (queBuffer)
{
for (int i = 0; i < iData; i++)
{
queBuffer.Enqueue(rgdi[i]);
rgdi[i] = null;
itemCount++;
}
/*if (fiter != 0) //there is a partially completed data item
{
rgdi[0] = rgdi[iData];
rgdi[iData] = null;
Console.WriteLine("I am here, fiter: {0} nsClient.DataAvailable: {1} iData: {2}", fiter, nsClient.DataAvailable, iData);
}*/
iData = 0;
}
} while (!eof);
time.Stop();
nsClient.Close();
if (rgdi.Length == 0 || (rgdi[rgdi.Length - 1] != null && !rgdi[rgdi.Length - 1].EOF))
{
lock (queBuffer)
{
DataItem di = new DataItem(2, null);
di.EOF = true;
queBuffer.Enqueue(di);
}
}
}
internal void GBMINStep(DataItem di, List<DataItem> ldi)
{
Group g;
if (FindGroup(di, out g))
{
//Unhandled Exception will occur if the value is not comparable
IComparable c = (IComparable)g.oResult;
if (c.CompareTo(di[val]) > 0)
g.oResult = di[val];
}
else
g.oResult = di[val];
}
internal void GBFinal(List<DataItem> ldi)
{
if (fComplete)
return;
fComplete = true;
DataItem di;
if (attrs == null)
{
di = new DataItem(1, ReleaseDataItem);
di.AddValue(gTotal.oResult);
ldi.Add(di);
}
else
{
foreach (int o in ht.Keys)
{
List<Group> al = (List<Group>)ht[o];
for (int i = 0; i < al.Count; i++)
{
Group g = al[i];
BuildGroupResult(g, out di);
ldi.Add(di);
}
}
ht.Clear();
}
}
internal void GBSUMStep(DataItem di, List<DataItem> ldi)
{
Group g;
if (FindGroup(di, out g))
g.oResult = (double)g.oResult + (double)di[val];
else
g.oResult = di[val];
}
private void BuildGroupResult(Group g, out DataItem di)
{
di = GetDataItem(1)[0];
foreach (object v in g.keys)
di.AddValue(v);
di.AddValue(g.oResult);
}
internal void PrjStep(DataItem di, List<DataItem> ldi)
{
ldi.Add(map(di));
}
protected void StrReader()
{
const int STREAMDATABUFFER = 100;
bool eof = false;
BinaryFormatter bf = new BinaryFormatter();
DataItem[] rgdi = new DataItem[STREAMDATABUFFER];
int iData = 0, fiter = 0;
do
{
while (iData < STREAMDATABUFFER && nsClient.DataAvailable && !eof)
{
try
{
rgdi[iData] = new DataItem(NUMBEROFINTS, null);
while (fiter < NUMBEROFINTS && !eof) //TODO: this will lock out if an item is not completed, we may still want to push previously completed items
{
if (nsClient.DataAvailable)
{
byte[] inbytes = new byte[4];
nsClient.Read(inbytes, 0, 4); //read in the bytes
int inint = BitConverter.ToInt32(inbytes, 0);
if (inint != -1)
rgdi[iData].AddValue(inint);
fiter++;
}
}
}
catch (System.IO.IOException)
{
//Assume that the connection was closed (though I have no way of knowing...)
eof = true;
}
catch (System.Exception ex)
{
Console.WriteLine("Something bad happened when we tried to read from the stream: {0} ({1})",
ex.Message, ((ex.InnerException != null) ? ex.InnerException.Message : ""));
eof = true;
}
iData++;
fiter = 0;
}
lock (queBuffer)
{
for (int i = 0; i < iData; i++)
{
queBuffer.Enqueue(rgdi[i]);
rgdi[i] = null;
}
iData = 0;
}
} while (!eof);
nsClient.Close();
if (rgdi.Length == 0 || (rgdi[rgdi.Length - 1] != null && !rgdi[rgdi.Length - 1].EOF))
{
lock (queBuffer)
{
DataItem di = new DataItem(2, null);
di.EOF = true;
queBuffer.Enqueue(di);
}
}
}
/// <summary>
/// Generic Map function that outputs results based on given attribute list
/// </summary>
/// <returns>Projected data item</returns>
/// <seealso cref="UnaryOp.Map"/>
private DataItem mapGeneric(DataItem di)
{
DataItem diRet = new DataItem(attrs.Length, null);
foreach (int i in attrs)
diRet.AddValue(di[i]);
return diRet;
}
/// <summary>
/// Combine two DataItem objects (in cross-product fashion)
/// </summary>
/// <param name="diL">Left DataItem object</param>
/// <param name="diR">Right DataItem object</param>
/// <param name="gdi">GetDataItem function</param>
/// <returns>a data item which is a pair of the given data items</returns>
public static DataItem Pair(DataItem diL, DataItem diR, Utility.DataItemPool.GetDataItem gdi)
{
//Call the correct constructor to create a DataItem or Punctuation
DataItem di;
if (diL is Punctuation && diR is Punctuation)
di = new Punctuation(diL.Count + diR.Count);
else
di = gdi(1)[0];
for (int i = 0; i < diL.Count; i++)
di.AddValue(diL[i]);
for (int i = 0; i < diR.Count; i++)
di.AddValue(diR[i]);
return di;
}
public DataItem NSSCheckSeg(DataItem di)
{
//<Time, VID, Spd, Xway, Lane, Dir, Seg, Pos>
int[] attrs = { 0, 1, 3, 4, 5, 6 };
if (di is Punctuation)
{
Punctuation pRet = new Punctuation(attrs.Length + 1);
foreach (int i in attrs)
pRet.AddValue(di[i]);
if (pRet[1] is Punctuation.LiteralPattern) //an end of vehicle punctuation
{
NSSTable.Remove(Convert.ToUInt32(((Punctuation.LiteralPattern)pRet[1]).Value));
}
pRet.AddValue(new Punctuation.WildcardPattern());
return pRet;
}
else
{
DataItem diRet = new DataItem(attrs.Length + 1, null);
foreach (int i in attrs)
diRet.AddValue(di[i]);
diRet.EOF = di.EOF;
diRet.TimeStamp = di.TimeStamp;
uint VID = Convert.ToUInt32(diRet[1]);
uint Seg = Convert.ToUInt32(diRet[5]);
if (NSSTable.ContainsKey(VID))
{
uint OldSeg = NSSTable[VID];
if (OldSeg == Seg)
diRet.AddValue((int)0);
else
{
diRet.AddValue((int)1);
NSSTable[VID] = Seg;
}
}
else
{
NSSTable.Add(VID, Seg);
diRet.AddValue((uint)1);
}
return diRet;
}
//<Time, VID, Xway, Lane, Dir, Seg, ?isNSS>
}