public void UpsertPayloadBuilder_FetchFrames_Expect_Ordered_Valid_Entries()
        {
            IDev2DataListUpsertPayloadBuilder <string> tmp = Dev2DataListBuilderFactory.CreateStringDataListUpsertBuilder();

            tmp.Add("[[scalar]]", "zzz");
            tmp.FlushIterationFrame();
            tmp.Add("[[scalar2]]", "aaa");

            IList <string> expressions = new List <string>();
            IList <string> values      = new List <string>();

            IList <IDataListPayloadIterationFrame <string> > frames = tmp.FetchFrames();

            int frameID = 1;

            foreach (IDataListPayloadIterationFrame <string> f in frames)
            {
                while (f.HasData())
                {
                    DataListPayloadFrameTO <string> t2 = f.FetchNextFrameItem();

                    expressions.Add(t2.Expression + "." + frameID);
                    values.Add(t2.Value + "." + frameID);
                }

                frameID++;
            }

            IList <string> expectedExpressions = new List <string> {
                "[[scalar]].1", "[[scalar2]].2"
            };


            CollectionAssert.AreEqual(expectedExpressions.ToArray(), expressions.ToArray());
            CollectionAssert.AreEqual(expectedExpressions.ToArray(), expressions.ToArray());
        }
        // 2 seconds for 10k entries with 75 columns ;)
        public void FlushIterations(PayloadIterationFrame <string> scopedFrame, bool isFramed, bool isTerminalFlush)
        {
            string error;

            Dev2TokenConverter tc = new Dev2TokenConverter();

            bool amendedData = false;

            // We do not care about data language in these cases, skip all the junk and get it done son ;)

            while (scopedFrame.HasData())
            {
                DataListPayloadFrameTO <string> tmp = scopedFrame.FetchNextFrameItem();
                string exp = tmp.Expression; //.Replace("(*)", "()"); // force conversion ;)
                string val = tmp.Value;

                // correction, we now need to support recursive evaluation ;(
                if (!DataListUtil.IsRootVariable(exp))
                {
                    ErrorResultTO errors;
                    var           tmpToken = _c.Evaluate(_bdl.UID, enActionType.User, exp, true, out errors);
                    if (errors.HasErrors())
                    {
                        throw new Exception(errors.MakeDisplayReady());
                    }
                    var scalar = tmpToken.FetchScalar();
                    if (scalar == null)
                    {
                        // ReSharper disable RedundantAssignment
                        exp = null;
                        // ReSharper restore RedundantAssignment
                    }
                    exp = tmpToken.FetchScalar().TheValue;
                }

                IIntellisenseResult token = tc.ParseTokenForMatch(exp, _bdl.FetchIntellisenseParts());

                if (token != null)
                {
                    // Get rs and field
                    string rs    = token.Option.Recordset;
                    string field = token.Option.Field;
                    string idx   = token.Option.RecordsetIndex;

                    if (rs != _lastRs && !string.IsNullOrEmpty(rs))
                    {
                        // Flush any existing row data for a different recordset ;)
                        if (_rowData != null)
                        {
                            if (!token.Option.IsScalar)
                            {
                                DumpColAtATime();
                            }

                            amendedData = false;
                        }

                        _bdl.TryGetEntry(rs, out _entry, out error);
                        if (error != string.Empty || _entry == null)
                        {
                            throw new Exception("Upsert Exception : " + error);
                        }

                        // stash last rs
                        _lastRs = rs;

                        // build new row data
                        int cnt = _entry.Columns.Count;
                        _rowData = new List <IBinaryDataListItem>(cnt);
                        InitRowBuffer(cnt);
                    }


                    if (!token.Option.IsScalar)
                    {
                        // set commit flag ;)
                        amendedData = true;

                        int colIdx = _entry.InternalFetchColumnIndex(field);

                        IBinaryDataListItem itm = _rowData[colIdx];

                        enRecordsetIndexType idxType = DataListUtil.GetRecordsetIndexTypeRaw(idx);

                        int tmpIdx = _dris.FetchRecordsetIndex(token, _entry, isFramed);

                        if (tmpIdx != _upsertIdx)
                        {
                            // silly users making algorithms slow ;(
                            // we need to dump data at this point... 1 fliping column at a time
                            DumpColAtATime();
                        }

                        _upsertIdx = tmpIdx;

                        if (_upsertIdx == 0)
                        {
                            throw new Exception("Invalid recordset index of 0");
                        }

                        // if numeric fetch the index
                        if (idxType == enRecordsetIndexType.Numeric)
                        {
                            Int32.TryParse(idx, out _upsertIdx);
                        }


                        itm.UpdateIndex(_upsertIdx);
                        itm.UpdateField(field);
                        itm.UpdateValue(val);

                        if (_rowData == null)
                        {
                            throw new Exception("Invalid Bulk Load Data");
                        }

                        _rowData[colIdx] = itm;
                    }
                    else
                    {
                        IBinaryDataListItem itm = DataListConstants.baseItem.Clone();

                        // else scalar and we need to get the entry ;(
                        IBinaryDataListEntry scalarEntry;
                        _bdl.TryGetEntry(field, out scalarEntry, out error);
                        itm.UpdateField(field);
                        itm.UpdateValue(val);

                        scalarEntry.TryPutScalar(itm, out error);

                        if (error != string.Empty)
                        {
                            throw new Exception(error);
                        }
                    }
                }
                else
                {
                    throw new Exception("Null token for [ " + exp + " ]");
                }
            }

            // flush the rowData out ;)
            if (_entry != null && _entry.IsRecordset && amendedData)
            {
                _entry.TryPutRecordRowAt(_rowData, _upsertIdx, out error);
                if (error != string.Empty)
                {
                    throw new Exception(error);
                }

                _dris.MoveIndexesToNextPosition();
            }

            if (isTerminalFlush)
            {
                ErrorResultTO errors;
                _c.PushBinaryDataListInServerScope(_liveFlushingLocation, _bdl, out errors);
            }

            // clear out the buffer
            ClearRowBuffer();
        }