Пример #1
0
        public HqlLine ReadLine()
        {
            HqlDataSource src = _sources[0];
            HqlLine       l   = src.ReadLine();

            return(l);
        }
Пример #2
0
        private void InitializeSourceForStreaming(HqlDataSource src)
        {
            if (src.OnJoin == null)
            {
                src.InitJoining();
            }
            src.LoadFieldReferences(_select.FieldGroup);
            src.LoadFieldReferences(_where.EvaluationCriteria);
            src.LoadFieldReferences(_groupby.FieldGroup);
            src.LoadFieldReferences(_orderby.FieldGroup);
            if (_settings.PrintCategorizeFilename)
            {
                src.LoadFieldReferences(_settings.Output.FieldGroup);
            }

            for (int j = 0; j < _from.Count; ++j)
            {
                if (_from[j].OnJoin != null)
                {
                    src.LoadFieldReferences(_from[j].OnJoin.EvaluationCriteria);
                }
            }

            src.BuildBlankValues();
        }
Пример #3
0
        private bool GetLineMultifileStreamable()
        {
            if (!_loadedFirstFiles)
            {
                GetLineMultifileStreamableLoadFirstFiles();
                _loadedFirstFiles = true;
            }

            HqlDataSource prior = _from[_from.Count - 2]; // guaranteed to be present b/c there is more than one file
            HqlDataSource src   = _from[_from.Count - 1];

            if (!_initializedForStreaming)
            {
                InitializeSourceForStreaming(src);
                _initializedForStreaming = true;
            }

            if (src.ReadThroughUsingJoin(prior, prior.OnJoin, _where, true))
            {
                _line = _select.Evaluate(src.ResultValues);
                return(true);
            }

            return(false);
        }
Пример #4
0
 private void SanityCheckRownum()
 {
     for (int i = 0; i < _from.Count; ++i)
     {
         HqlDataSource src = _from[i];
         if (src.OnJoin != null && src.OnJoin.ContainsRownum(true))
         {
             throw new Exception("Cannot have a \"rownum\" without a table in \"ON Clause\" due to complexity");
         }
     }
 }
Пример #5
0
        public HqlValuesComparer(HqlDataSource prior, HqlWhere onJoin, HqlWhere where) //, HqlWith settings)
        {
            _prior  = prior;
            _onJoin = onJoin;
            _where  = where;
            //_settings = settings;
            _orderOfSortFields = new ArrayList();
            _mode = HqlCompareMode.SORT_MODE;

            _onJoin.CreateSortFields(this);
            _where.CreateSortFields(this);

            _currentPrior = -1;
            _joinMethod   = HqlJoinMethod.SORT_COMPARE;
        }
Пример #6
0
 private void AddSource(HqlDataSource source)
 {
     if (_sources == null)
     {
         _sources    = new HqlDataSource[1];
         _sources[0] = source;
     }
     else
     {
         HqlDataSource[] newfields = new HqlDataSource[_sources.Length + 1];
         for (int i = 0; i < _sources.Length; ++i)
         {
             newfields[i] = _sources[i];
         }
         newfields[_sources.Length] = source;
         _sources = newfields;
     }
 }
Пример #7
0
        private void CalculateDataMultifiles()
        {
            HqlDataSource prior = _from[_from.Count - 2]; // guaranteed to be present b/c there is more than one file
            HqlDataSource src   = _from[_from.Count - 1];

            InitializeSourceForStreaming(src);

            for (; ; _linenum++)
            {
                if (src.ReadThroughUsingJoin(prior, prior.OnJoin, _where, true))
                {
                    CreateKeyAndValue(src.ResultValues);
                }
                else
                {
                    prior.DropLoadedData();
                    break;
                }
            }
        }
Пример #8
0
        private void GetLineMultifileStreamableLoadFirstFiles()
        {
            HqlDataSource prior = null;

            for (int i = 0; i < _from.Count - 1; ++i)
            {
                HqlDataSource src = _from[i];
                InitializeSourceForStreaming(src);

                if (i == 0)
                {
                    src.ReadThroughUsingJoin(null, src.OnJoin, _where, false);
                }
                else
                {
                    src.ReadThroughUsingJoin(prior, prior.OnJoin, _where, false);
                    prior.DropLoadedData();
                }

                prior = src;
            }
        }
Пример #9
0
        public void Parse(HqlTokenProcessor processor)
        {
            HqlToken   token;
            bool       FileExpected = true;
            HqlKeyword joinType;

            // first thing should be from
            token = processor.GetToken();
            if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.FROM)
            {
                throw new Exception("Expected FROM");
            }

            bool ReadNextToken = true;

            // now pull back things until you get to EOF or WHERE
            for (; ;)
            {
                joinType = HqlKeyword.UNKNOWN;

                if (ReadNextToken)
                {
                    processor.MoveNextToken();
                }
                ReadNextToken = true;

                token = processor.GetToken();
                if (processor.MatchesEndOfProcessing())
                {
                    break;
                }

                if (token.WordType == HqlWordType.KEYWORD &&
                    (
                        token.Keyword == HqlKeyword.WHERE ||
                        token.Keyword == HqlKeyword.GROUPBY ||
                        token.Keyword == HqlKeyword.HAVING ||
                        token.Keyword == HqlKeyword.ORDERBY
                    )
                    )
                {
                    break;
                }

                if (token.WordType == HqlWordType.KEYWORD &&
                    (
                        token.Keyword == HqlKeyword.FULL_OUTER ||
                        token.Keyword == HqlKeyword.LEFT_OUTER ||
                        token.Keyword == HqlKeyword.RIGHT_OUTER ||
                        token.Keyword == HqlKeyword.INNER
                    )
                    )
                {
                    joinType = token.Keyword;

                    processor.MoveNextToken();
                    token = processor.GetToken();
                    if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.JOIN)
                    {
                        throw new Exception("Expected JOIN keyword");
                    }
                    FileExpected = true;

                    processor.MoveNextToken();
                    token = processor.GetToken();
                }

                // else time to process it!
                if (!FileExpected)
                {
                    break;
                }

                FileExpected = false;

                if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.STDIN)
                {
                    AddStdin();
                }
                else if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.DUAL)
                {
                    AddDual();
                }
                else if (token.WordType == HqlWordType.TEXT || token.WordType == HqlWordType.LITERAL_STRING)
                {
                    AddFile(token.Data);
                }
                else if (token.WordType == HqlWordType.STREAM)
                {
                    HqlDataSource source = new HqlDataSource(HqlDataSourceType.STREAM, (System.IO.StreamReader)token.Parsed);
                    AddSource(source);
                }
                else if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.OPENPAREN)
                {
                    // NESTED QUERY!
                    HqlStream cs           = new HqlStream(processor.GetRemainingSql(false), ")");
                    string    remainingSql = cs.RemainingSql;
                    processor.SetRemainingSql(remainingSql);
                    processor.MoveNextToken();
                    token = processor.GetToken();
                    if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.CLOSEDPAREN)
                    {
                        throw new Exception("Nested query SQL did not return correctly");
                    }

                    AddTextReader(cs);
                }
                else
                {
                    throw new Exception("Unknown type of FROM clause");
                }

                // look to process SKIP
                for (;;)
                {
                    bool breakOut = false;

                    if (ReadNextToken)
                    {
                        processor.MoveNextToken();
                        token = processor.GetToken();
                    }

                    if (_sources != null && _sources.Length > 0 && (token.WordType == HqlWordType.UNKNOWN || token.WordType == HqlWordType.KEYWORD))
                    {
                        HqlToken option;
                        switch (token.Data.ToUpper())
                        {
                        case "SKIP":
                        {
                            option = processor.GetOptionData(token.Data);
                            if (option.WordType != HqlWordType.INT || option.ParsedAsInt <= 0)
                            {
                                throw new Exception("Skip count must be greater than zero");
                            }
                            CurrentSource.SkipRecords = option.ParsedAsInt;
                            ReadNextToken             = true;
                            break;
                        }

                        case "DELIMITER":
                        case "DELIM":
                        {
                            option = processor.GetOptionData(token.Data);
                            if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
                            {
                                throw new Exception(String.Format("Expected a valid delimiter after {0}", token.Data));
                            }
                            CurrentSource.Delimiter = HqlTokenProcessor.CleanupDelimiter(option.Data);
                            ReadNextToken           = true;
                            break;
                        }

                        case "AS":
                        {
                            option = processor.GetOptionData(token.Data);
                            if (option.WordType != HqlWordType.UNKNOWN)
                            {
                                throw new Exception("Expected a table reference name following AS");
                            }
                            CurrentSource.TableReference = option.Data;
                            break;
                        }

                        default:
                            breakOut = true;
                            break;
                        }
                    }
                    else
                    {
                        breakOut = true;
                    }

                    if (breakOut)
                    {
                        ReadNextToken = false;
                        break;
                    }
                }


                if (joinType != HqlKeyword.UNKNOWN)
                {
                    HqlDataSource src = LeftSideOfJoin;
                    src.JoinType = joinType;

                    if (ReadNextToken)
                    {
                        processor.MoveNextToken();
                        token = processor.GetToken();
                    }

                    if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.ON)
                    {
                        throw new Exception("Expected ON following a table");
                    }

                    src.InitJoining();
                    src.OnJoin.Parse(processor, HqlKeyword.ON, HqlKeyword.ON, null);
                    ReadNextToken = false;
                }
            }

            if (FileExpected)
            {
                throw new Exception("Expected additional FROM fields");
            }
        }
Пример #10
0
        private void AddTextReader(HqlStream stream)
        {
            HqlDataSource source = new HqlDataSource(HqlDataSourceType.HQLQUERY, stream);

            AddSource(source);
        }
Пример #11
0
        private void AddDual()
        {
            HqlDataSource source = new HqlDataSource(HqlDataSourceType.DUAL, String.Empty);

            AddSource(source);
        }
Пример #12
0
        private void AddStdin()
        {
            HqlDataSource source = new HqlDataSource(HqlDataSourceType.STDIN, String.Empty);

            AddSource(source);
        }
Пример #13
0
        ///////////////////////
        // Private

        private void AddFile(string name)
        {
            HqlDataSource source = new HqlDataSource(HqlDataSourceType.FILE, name);

            AddSource(source);
        }
Пример #14
0
        //public HqlValues MergeValues(HqlValues priorvalues, HqlValues thisvalues, ref HqlFieldGroup combinedFields)
        //{
        //    if (combinedFields == null)
        //        combinedFields = new HqlFieldGroup(priorvalues.FieldsImpacted, thisvalues.FieldsImpacted);
        //    return new HqlValues(combinedFields, priorvalues, thisvalues);
        //}

        public bool ReadThroughUsingJoin(HqlDataSource prior, HqlWhere onJoin, HqlWhere where, bool selectRecord)
        {
            // TODO
            // - need to sort the prior how I'm going to join these.
            // - read through current and join them and save to new object
            // - clear all the saved data from the prior object
            //HqlFieldGroup combinedFieldsImpacted = null;
            //HqlValuesComparer comparer = prior.comparer;

            // TODO, sort the prior
            //bool sortedPrior = false;

            for (; ;)
            {
                if (selectRecord && Lines.Count > 0)
                {
                    _resultValues = Lines[0];
                    Lines.RemoveAt(0);

                    if (where.Evaluate(_resultValues))
                    {
                        return(true);
                    }
                    continue;
                }

                if (EndOfStream)
                {
                    if (!_completedJoin)
                    {
                        _completedJoin = true;

                        if (prior != null && (prior.JoinType == HqlKeyword.LEFT_OUTER || prior.JoinType == HqlKeyword.FULL_OUTER))
                        {
                            prior.Comparer.JoinMethod = HqlJoinMethod.RETURN_ALL;
                            prior.Comparer.Reset();
                            prior.comparer.SetValues(BlankValues);

                            for (; prior.Comparer.MoveNext();)
                            {
                                prior.Comparer.Current.IncreaseUsedCount();
                                HqlValues combinedvalues = prior.Comparer.CombinedValues;
                                SaveRowValue(combinedvalues);
                            }

                            continue;
                        }
                    }

                    _resultValues = null;
                    return(false);
                }

                //string fromline = ReadLine();
                //if (fromline.Equals(String.Empty) && EndOfStream)
                //    continue;
                //HqlLine line = new HqlLine(fromline, settings, RowNum)
                HqlLine line = ReadLine();
                if (line.IsBlank && EndOfStream)
                {
                    continue;
                }

#if DEBUG
                if (line.Line.StartsWith("1503|706"))
                {
                    int stop = 0;
                }
#endif

                // get the current line of data
                HqlValues thisvalues = new HqlValues(FieldsImpacted, line);

                // does this match the where statement?
                if (JoinType == HqlKeyword.LEFT_OUTER || JoinType == HqlKeyword.FULL_OUTER)
                {
                    // pass through, don't evaluate the ON command because I want this regardless
                }
                else if (prior != null && (prior.JoinType == HqlKeyword.FULL_OUTER || prior.JoinType == HqlKeyword.RIGHT_OUTER))
                {
                    // pass through
                }
                else if (!onJoin.EvaluateJoin(thisvalues, true)) // || !where.EvaluateJoin(thisvalues))
                {
                    // didn't pass through, so continue
                    continue;
                }

                // This is here for the first "join"
                // I need to save all the records.
                if (prior == null)
                {
                    SaveRowValue(thisvalues);
                    continue;
                }

                // now time to join the data!
                bool savedThisRecord = false;

                // do preprocessing if this is to be searched through sorting
                if (prior.Comparer == null)
                {
                    if (prior.Lines.Count == 0)
                    {
                        throw new Exception("Data integrity issue. Didn't find anything to join with on prior! TODO, this can be valid, need to fix!!");
                    }

                    HqlValues priorvalues = prior.Lines[0];                       // guaranteed to be there
                    prior.Comparer = new HqlValuesComparer(prior, onJoin, where); //, settings);
                    prior.Comparer.CombinedFieldsImpacted = new HqlFieldGroup(priorvalues.FieldsImpacted, thisvalues.FieldsImpacted);

                    if (prior.Comparer.CountOfJoinFields == 0)
                    {
                        prior.Comparer.JoinMethod = HqlJoinMethod.LINEAR_COMPARE;
                    }

                    Console.Error.WriteLine(String.Format("Using joining of {0} between ({1}) {2} and ({3}) {4}",
                                                          prior.Comparer.JoinMethod.ToString(),
                                                          prior.TableReference,
                                                          prior.SourceName,
                                                          this.TableReference,
                                                          this.SourceName)
                                            );

                    if (prior.Comparer.JoinMethod == HqlJoinMethod.SORT_COMPARE)
                    {
                        prior.Comparer.SortPrior();
                    }
                }

                prior.Comparer.SetValues(thisvalues);
                prior.Comparer.Reset();

                for (; prior.Comparer.MoveNext();)
                {
                    prior.Comparer.Current.IncreaseUsedCount();

                    HqlValues combinedvalues = prior.Comparer.CombinedValues;

                    SaveRowValue(combinedvalues);
                    savedThisRecord = true;
                }

                if (!savedThisRecord && (prior.JoinType == HqlKeyword.RIGHT_OUTER || prior.JoinType == HqlKeyword.FULL_OUTER))
                {
                    HqlValues combinedvalues = new HqlValues(prior.Comparer.CombinedFieldsImpacted, prior.BlankValues, thisvalues);
                    SaveRowValue(combinedvalues);
                }
            }
        }