/// <summary>
/// Function to handle punctuations as they arrive in the eddy.
/// In this case, we simply enqueue the punctuation as if it were
/// a data item, and the Iterate function for each operator will
/// process it appropriately
/// </summary>
/// <param name="p">The incoming punctuation</param>
public void EKeep(Punctuation p)
{
//All we need to do is treat the punctuation as if it were a data item.
// Just enqueue it, and let the Iterate function for each participating
// operator process it as usual.
queueDataPool.Enqueue(new EddyDataItems(p, listOperators.Count));
}
internal void SelProp(Punctuation p, List<DataItem> ldi)
{
ldi.Add(p);
}
internal void PrjProp(Punctuation p, List<DataItem> ldi)
{
//Potentially expensive to do this for every punctuation.
// Adding a describe operator at the base of the query plan may improve
// performance (by eliminating the check here).
if (p.Describes(attrs))
ldi.Add(map(p));
}
/// <summary>
/// Propagation functionality for inputs of union
/// </summary>
/// <param name="p">Punctuation from the input</param>
/// <param name="ldi">Any punctuations that can be output</param>
/// <seealso cref="QueryEngine.Prop"/>
public void UProp(Punctuation p, List<DataItem> ldi)
{
bool fCoerced = false;
Punctuation pNew = null, pProp;
//First, coerce this punctuation with those seen on this input before
for (int i = 0; i < liPuncts[CurrentInput].Count && !fCoerced; i++)
{
Punctuation pTmp = Punctuation.Coerce(p, liPuncts[CurrentInput][i]);
if (pTmp != null)
{
liPuncts[CurrentInput][i] = pTmp;
fCoerced = true;
pNew = pTmp;
}
}
if (!fCoerced)
{
liPuncts[CurrentInput].Add(p);
pNew = p;
}
//Now, see if we can combine a punctuation from punctuations from all inputs
pProp = CombineFromLists(liPuncts, 0, pNew);
if (pProp != null)
{
//Before we output, we need to remove the combined punctuation from the existing
// punctuations, so that they don't grow huge
for (int i = 0; i < liPuncts.Length; i++)
{
for (int j = 0; j < liPuncts[i].Count; j++)
{
Punctuation pUpdate = Punctuation.Uncoerce(liPuncts[i][j], pProp);
if (pUpdate == null)
liPuncts[i].RemoveAt(j--);
else
liPuncts[i][j] = pUpdate;
}
}
ldi.Add(pProp);
}
}
/// <summary>
/// Trivial prop functionality : ignore the input continue
/// </summary>
/// <param name="p">The input punctuation</param>
/// <param name="ldi">The output punctuations due to this punctuation</param>
public static void PropTrivial(Punctuation p, List<DataItem> ldi)
{
}
internal void GBProp(Punctuation p, List<DataItem> ldi)
{
bool fEOF = true;
for (int i = 0; fEOF && i < p.Count; i++)
{
fEOF &= (p[i] is Punctuation.WildcardPattern);
}
if (fEOF && attrs == null)
{
Punctuation pOut = new Punctuation(1);
pOut.AddValue(new Punctuation.WildcardPattern());
ldi.Add(pOut);
}
else if (fEOF || (attrs != null && p.Describes(attrs)))
{
Punctuation pOut = new Punctuation(attrs.Length + 1);
foreach (int a in attrs)
pOut.AddValue(p[a]);
pOut.AddValue(new Punctuation.WildcardPattern());
ldi.Add(pOut);
}
}
internal void GBKeep(Punctuation p)
{
if (attrs != null)
{
List<Group> al = FindMatchingGroups(p);
if (al != null)
{
for (int i = 0; i < al.Count; i++)
{
int hc = al[i].hashcode;
List<Group> alCached = ht[hc];
alCached.Remove(al[i]);
foreach (object obj in al[i].keys)
(obj as DataItem).Dispose();
if (alCached.Count == 0)
ht.Remove(hc);
}
}
}
}
private void SProp(Punctuation p, List<DataItem> ldi)
{
//TODO: Need to fix this to ensure the punctuation describes
// a prefix of the sorted output, and not just describes the
// sort attributes
if (p.Describes(attrs))
ldi.Add(p);
}
/// <summary>
/// Add a pattern to the Scheme
/// </summary>
/// <param name="pattern">The pattern to add to the Scheme</param>
public void AddPattern(Punctuation.Pattern pattern)
{
if (scheme.Count >= max_size) return;
if (pattern == null) return;
scheme.Add(pattern);
}
private List<Punctuation> Flatten(Punctuation p, int iAttr, List<Punctuation> ret)
{
if (iAttr == p.Count)
{
return ret;
}
else
{
if (p[iAttr] is Punctuation.RangePattern)
return null;
else if (p[iAttr] is Punctuation.LiteralPattern || p[iAttr] is Punctuation.WildcardPattern)
return Flatten(p, iAttr + 1, ret);
else //ListPattern
{
for (int i = 0; i < ret.Count; i++)
{
for (int j = 0; j < ((ListPattern)ret[i][iAttr]).Values.Length; j++)
{
Punctuation pRet = ret[i].Copy();
pRet[iAttr] = new LiteralPattern(((ListPattern)ret[i][iAttr]).Values[j]);
ret.Insert(i + j + 1, pRet);
}
ret.RemoveAt(i);
i += ((ListPattern)p[iAttr]).Values.Length - 1;
}
return Flatten(p, iAttr + 1, ret);
}
}
}
public DataItem PreProcess(DataItem di)
{
//<Time, VID, Spd, Xway, Lane, Dir, Seg, Pos>
if ((int)di[1] == -1) //we need to turn it into a proper punctuation
{
const int WildCards = 7;
Punctuation p = new Punctuation(WildCards + 1);
p.AddValue(new Punctuation.LiteralPattern(di[0]));
for (int i = 0; i < WildCards; i++)
p.AddValue(new Punctuation.WildcardPattern());
return p;
}
else
return di;
//<Time, VID, Spd, Xway, Lane, Dir, Seg, Pos>
}
public DataItem NSSCheckSeg(DataItem di)
{
//<Time, VID, Spd, Xway, Lane, Dir, Seg, Pos>
int[] attrs = { 0, 1, 3, 4, 5, 6 };
if (di is Punctuation)
{
Punctuation pRet = new Punctuation(attrs.Length + 1);
foreach (int i in attrs)
pRet.AddValue(di[i]);
if (pRet[1] is Punctuation.LiteralPattern) //an end of vehicle punctuation
{
NSSTable.Remove(Convert.ToUInt32(((Punctuation.LiteralPattern)pRet[1]).Value));
}
pRet.AddValue(new Punctuation.WildcardPattern());
return pRet;
}
else
{
DataItem diRet = new DataItem(attrs.Length + 1, null);
foreach (int i in attrs)
diRet.AddValue(di[i]);
diRet.EOF = di.EOF;
diRet.TimeStamp = di.TimeStamp;
uint VID = Convert.ToUInt32(diRet[1]);
uint Seg = Convert.ToUInt32(diRet[5]);
if (NSSTable.ContainsKey(VID))
{
uint OldSeg = NSSTable[VID];
if (OldSeg == Seg)
diRet.AddValue((int)0);
else
{
diRet.AddValue((int)1);
NSSTable[VID] = Seg;
}
}
else
{
NSSTable.Add(VID, Seg);
diRet.AddValue((uint)1);
}
return diRet;
}
//<Time, VID, Xway, Lane, Dir, Seg, ?isNSS>
}
public void InsertPuncts(DateTime sysTime)
{
// insert new time punctuations
Punctuation p = new Punctuation(TYPE0);
for (int i = 0; i < TYPE0; i++)
{
if (i != INDEXOFTIME)
p.AddValue(new Punctuation.WildcardPattern());
else
p.AddValue(new Punctuation.LiteralPattern((ulong)TimeSpan.FromTicks(sysTime.Ticks).TotalSeconds));
}
lrcs[0].sendData(p);
// check for expiredVehichles
List<uint> removeVids = new List<uint>();
foreach(KeyValuePair<uint, DateTime> kv in vids)
{
if (sysTime.Subtract(kv.Value).TotalSeconds > 30) //it's been longer than 30 seconds
{
removeVids.Add(kv.Key);
}
}
foreach (uint val in removeVids)
{
Punctuation p2 = new Punctuation(TYPE0);
for (int j = 0; j < TYPE0; j++)
{
if (j != INDEXOFVID)
p2.AddValue(new Punctuation.WildcardPattern());
else
p2.AddValue(new Punctuation.LiteralPattern((uint)val));
}
lrcs[0].sendData(p2);
vids.Remove(val);
}
removeVids.Clear();
}
private Punctuation CombineFromLists(List<Punctuation>[] rgliPunct, int iCurrent, Punctuation pCombine)
{
if (iCurrent == rgliPunct.Length)
return pCombine;
for (int i = 0; i < rgliPunct[iCurrent].Count && i != CurrentInput; i++)
{
Punctuation pTmp = Punctuation.Combine(pCombine, rgliPunct[iCurrent][i]);
if (pTmp != null)
{
pTmp = CombineFromLists(rgliPunct, iCurrent + 1, pTmp);
if (pTmp != null)
return pTmp;
}
}
//If we get out of the loop, then we couldn't combine punctutations to a valid punctuation.
return null;
}
private void SKeep(Punctuation p)
{
//TODO: Need to fix this to ensure the punctuation describes
// a prefix of the sorted output, and not just describes the
// sort attributes
if (p.Describes(attrs))
{
//Should we still have to do this, since we called SPass first?
al.Sort(cmpData);
int c;
for (c = 0; c < al.Count && p.Match(al[c]); c++) ;
al.RemoveRange(0, c);
}
}
private void SPass(Punctuation p, List<DataItem> ldi)
{
//TODO: Need to fix this to ensure the punctuation describes
// a prefix of the sorted output, and not just describes the
// sort attributes
if (p.Describes(attrs))
{
al.Sort(cmpData);
for (int i = 0; i < al.Count && p.Match(al[i]); i++)
ldi.Add(al[i]);
}
}
/// <summary>
/// Remove the given punctuation from this meta-punctuation
/// </summary>
/// <param name="p">The punctuation to remove</param>
public void Remove(Punctuation p)
{
bool fSimilar = false;
int i;
Punctuation pNew = new Punctuation(p.Count);
for (i = 0; i < p.Count; i++)
pNew.AddValue(new WildcardPattern());
for (i = 0; i < listP.Count && fSimilar == false; i++)
{
fSimilar = true;
for (int a = 0; a < p.Count && fSimilar; a++)
{
pNew[a] = ((Pattern)listP[i][a]).Uncoerce(((Pattern)p[a]));
fSimilar = !(pNew[a] is EmptyPattern);
}
}
i--;
if (fSimilar)
{
for (int a = 0; a < p.Count; a++)
listP[i][a] = pNew[a];
}
}
/// <summary>
/// Output new data items
/// </summary>
/// <returns>The DataItem objects to output</returns>
/// <seealso cref="Data.DataItem"/>
public override List<DataItem> Iterate(DataItemPool.GetDataItem gdi, DataItemPool.ReleaseDataItem rdi)
{
ldiBufferOut.Clear();
DataItem[] rgdi = gdi(CITERATE);
for (int i = 0; i < CITERATE && !eof; i++)
{
if (iRow < cRows)
{
rgdi[i].AddValue((int)((int)iRow / 7));
rgdi[i].AddValue((int)iRow);
}
else
{
eof = rgdi[i].EOF = true;
Punctuation p = new Punctuation(2);
p.AddValue(new Punctuation.WildcardPattern());
p.AddValue(new Punctuation.WildcardPattern());
ldiBufferOut.Add(p);
}
ldiBufferOut.Add(rgdi[i]);
if (iPunct == 0 && ((int)iRow / 7) != ((int)(iRow + 1) / 7))
{
//Let's add a punctuation
Punctuation p = new Punctuation(2);
p.AddValue(new Punctuation.LiteralPattern((int)(iRow / 7)));
p.AddValue(new Punctuation.WildcardPattern());
ldiBufferOut.Add(p);
}
if (iPunct == 1)
{
if (iRow >= CPUNCTLISTSIZE && iRow % CPUNCTLISTSIZE == 0)
{
Punctuation p2 = new Punctuation(2);
p2.AddValue(new Punctuation.WildcardPattern());
Punctuation.LiteralPattern[] rglp = new Punctuation.LiteralPattern[CPUNCTLISTSIZE];
for (int iLit = 0; iLit < CPUNCTLISTSIZE; iLit++)
rglp[iLit] = new Punctuation.LiteralPattern((int)iRow - CPUNCTLISTSIZE + iLit);
p2.AddValue(new Punctuation.ListPattern(rglp));
ldiBufferOut.Add(p2);
}
}
if (iPunct == 2)
{
if (iRow >= CPUNCTLISTSIZE && iRow % CPUNCTLISTSIZE == 0)
{
Punctuation p2 = new Punctuation(2);
p2.AddValue(new Punctuation.WildcardPattern());
int iMin = iRow - CPUNCTLISTSIZE + 1, iMax = iRow;
p2.AddValue(new Punctuation.RangePattern(iMin, iMax));
ldiBufferOut.Add(p2);
}
}
iRow++;
}
cRowsOutput += ldiBufferOut.Count;
return ldiBufferOut;
}
/// <summary>
/// Copy constructor for a punctuation
/// </summary>
/// <param name="p">The punctuation to copy</param>
/// <param name="c">The number of attributes more to track than the copied punctuation</param>
public Punctuation(Punctuation p, uint c)
: base(p, c, null)
{
}
internal void GBPass(Punctuation p, List<DataItem> ldi)
{
DataItem di;
bool fEOF = true;
for (int i = 0; fEOF && i < p.Count; i++)
fEOF &= (p[i] is Punctuation.WildcardPattern);
if (fEOF)
GBFinal(ldi);
else if (attrs != null)
{
//Can't ever output results early unless we're grouping on specific attributes
List<Group> al = FindMatchingGroups(p);
if (al != null)
{
for (int i = 0; i < al.Count; i++)
{
Group g = al[i];
BuildGroupResult(g, out di);
ldi.Add(di);
}
}
}
}
/// <summary>
/// The removal (difference) of the second punctuation from the first
/// </summary>
/// <param name="p1">The original punctuation</param>
/// <param name="p2">The punctuation to remove</param>
/// <returns>The punctuation that represents the difference of p1 and p2, or null of none exists</returns>
public static Punctuation Uncoerce(Punctuation p1, Punctuation p2)
{
if (p1.Count != p2.Count)
return null;
Punctuation pRet = new Punctuation(p1.Count);
for (int ip = 0; ip < p1.Count; ip++)
{
Pattern pat = ((Pattern)p1[ip]).Uncoerce((Pattern)p2[ip]);
if (pat is EmptyPattern)
return null;
else
pRet.AddValue(pat);
}
return pRet;
}
/// <summary>
/// Trivial keep functionality : ignore the input continue
/// </summary>
/// <param name="p">The input punctuation</param>
public static void KeepTrivial(Punctuation p)
{
}
/// <summary> Make a copy of this punctuation </summary>
/// <returns>a copy of this punctuation</returns>
public Punctuation Copy()
{
Punctuation ret = new Punctuation(this.Count);
for (int i = 0; i < this.Count; i++)
ret.AddValue(this[i]);
return ret;
}
/// <summary>
/// Combine two DataItem objects (in cross-product fashion)
/// </summary>
/// <param name="diL">Left DataItem object</param>
/// <param name="diR">Right DataItem object</param>
/// <param name="gdi">GetDataItem function</param>
/// <returns>a data item which is a pair of the given data items</returns>
public static DataItem Pair(DataItem diL, DataItem diR, Utility.DataItemPool.GetDataItem gdi)
{
//Call the correct constructor to create a DataItem or Punctuation
DataItem di;
if (diL is Punctuation && diR is Punctuation)
di = new Punctuation(diL.Count + diR.Count);
else
di = gdi(1)[0];
for (int i = 0; i < diL.Count; i++)
di.AddValue(diL[i]);
for (int i = 0; i < diR.Count; i++)
di.AddValue(diR[i]);
return di;
}
internal void DEKeep(Punctuation p)
{
List<int> keys = new List<int>();
foreach (int k in ht.Keys)
{
List<DataItem> al = ht[k];
for (int i = 0; i < al.Count; i++)
{
if (p.Match(al[i]))
al.RemoveAt(i--);
}
if (al.Count == 0)
keys.Add(k);
}
foreach (int k in keys)
ht.Remove(k);
}
