public HqlLine ReadLine()
{
HqlDataSource src = _sources[0];
HqlLine l = src.ReadLine();
return(l);
}
private void InitializeSourceForStreaming(HqlDataSource src)
{
if (src.OnJoin == null)
{
src.InitJoining();
}
src.LoadFieldReferences(_select.FieldGroup);
src.LoadFieldReferences(_where.EvaluationCriteria);
src.LoadFieldReferences(_groupby.FieldGroup);
src.LoadFieldReferences(_orderby.FieldGroup);
if (_settings.PrintCategorizeFilename)
{
src.LoadFieldReferences(_settings.Output.FieldGroup);
}
for (int j = 0; j < _from.Count; ++j)
{
if (_from[j].OnJoin != null)
{
src.LoadFieldReferences(_from[j].OnJoin.EvaluationCriteria);
}
}
src.BuildBlankValues();
}
private bool GetLineMultifileStreamable()
{
if (!_loadedFirstFiles)
{
GetLineMultifileStreamableLoadFirstFiles();
_loadedFirstFiles = true;
}
HqlDataSource prior = _from[_from.Count - 2]; // guaranteed to be present b/c there is more than one file
HqlDataSource src = _from[_from.Count - 1];
if (!_initializedForStreaming)
{
InitializeSourceForStreaming(src);
_initializedForStreaming = true;
}
if (src.ReadThroughUsingJoin(prior, prior.OnJoin, _where, true))
{
_line = _select.Evaluate(src.ResultValues);
return(true);
}
return(false);
}
private void SanityCheckRownum()
{
for (int i = 0; i < _from.Count; ++i)
{
HqlDataSource src = _from[i];
if (src.OnJoin != null && src.OnJoin.ContainsRownum(true))
{
throw new Exception("Cannot have a \"rownum\" without a table in \"ON Clause\" due to complexity");
}
}
}
public HqlValuesComparer(HqlDataSource prior, HqlWhere onJoin, HqlWhere where) //, HqlWith settings)
{
_prior = prior;
_onJoin = onJoin;
_where = where;
//_settings = settings;
_orderOfSortFields = new ArrayList();
_mode = HqlCompareMode.SORT_MODE;
_onJoin.CreateSortFields(this);
_where.CreateSortFields(this);
_currentPrior = -1;
_joinMethod = HqlJoinMethod.SORT_COMPARE;
}
private void AddSource(HqlDataSource source)
{
if (_sources == null)
{
_sources = new HqlDataSource[1];
_sources[0] = source;
}
else
{
HqlDataSource[] newfields = new HqlDataSource[_sources.Length + 1];
for (int i = 0; i < _sources.Length; ++i)
{
newfields[i] = _sources[i];
}
newfields[_sources.Length] = source;
_sources = newfields;
}
}
private void CalculateDataMultifiles()
{
HqlDataSource prior = _from[_from.Count - 2]; // guaranteed to be present b/c there is more than one file
HqlDataSource src = _from[_from.Count - 1];
InitializeSourceForStreaming(src);
for (; ; _linenum++)
{
if (src.ReadThroughUsingJoin(prior, prior.OnJoin, _where, true))
{
CreateKeyAndValue(src.ResultValues);
}
else
{
prior.DropLoadedData();
break;
}
}
}
private void GetLineMultifileStreamableLoadFirstFiles()
{
HqlDataSource prior = null;
for (int i = 0; i < _from.Count - 1; ++i)
{
HqlDataSource src = _from[i];
InitializeSourceForStreaming(src);
if (i == 0)
{
src.ReadThroughUsingJoin(null, src.OnJoin, _where, false);
}
else
{
src.ReadThroughUsingJoin(prior, prior.OnJoin, _where, false);
prior.DropLoadedData();
}
prior = src;
}
}
public void Parse(HqlTokenProcessor processor)
{
HqlToken token;
bool FileExpected = true;
HqlKeyword joinType;
// first thing should be from
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.FROM)
{
throw new Exception("Expected FROM");
}
bool ReadNextToken = true;
// now pull back things until you get to EOF or WHERE
for (; ;)
{
joinType = HqlKeyword.UNKNOWN;
if (ReadNextToken)
{
processor.MoveNextToken();
}
ReadNextToken = true;
token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
break;
}
if (token.WordType == HqlWordType.KEYWORD &&
(
token.Keyword == HqlKeyword.WHERE ||
token.Keyword == HqlKeyword.GROUPBY ||
token.Keyword == HqlKeyword.HAVING ||
token.Keyword == HqlKeyword.ORDERBY
)
)
{
break;
}
if (token.WordType == HqlWordType.KEYWORD &&
(
token.Keyword == HqlKeyword.FULL_OUTER ||
token.Keyword == HqlKeyword.LEFT_OUTER ||
token.Keyword == HqlKeyword.RIGHT_OUTER ||
token.Keyword == HqlKeyword.INNER
)
)
{
joinType = token.Keyword;
processor.MoveNextToken();
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.JOIN)
{
throw new Exception("Expected JOIN keyword");
}
FileExpected = true;
processor.MoveNextToken();
token = processor.GetToken();
}
// else time to process it!
if (!FileExpected)
{
break;
}
FileExpected = false;
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.STDIN)
{
AddStdin();
}
else if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.DUAL)
{
AddDual();
}
else if (token.WordType == HqlWordType.TEXT || token.WordType == HqlWordType.LITERAL_STRING)
{
AddFile(token.Data);
}
else if (token.WordType == HqlWordType.STREAM)
{
HqlDataSource source = new HqlDataSource(HqlDataSourceType.STREAM, (System.IO.StreamReader)token.Parsed);
AddSource(source);
}
else if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.OPENPAREN)
{
// NESTED QUERY!
HqlStream cs = new HqlStream(processor.GetRemainingSql(false), ")");
string remainingSql = cs.RemainingSql;
processor.SetRemainingSql(remainingSql);
processor.MoveNextToken();
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.CLOSEDPAREN)
{
throw new Exception("Nested query SQL did not return correctly");
}
AddTextReader(cs);
}
else
{
throw new Exception("Unknown type of FROM clause");
}
// look to process SKIP
for (;;)
{
bool breakOut = false;
if (ReadNextToken)
{
processor.MoveNextToken();
token = processor.GetToken();
}
if (_sources != null && _sources.Length > 0 && (token.WordType == HqlWordType.UNKNOWN || token.WordType == HqlWordType.KEYWORD))
{
HqlToken option;
switch (token.Data.ToUpper())
{
case "SKIP":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.INT || option.ParsedAsInt <= 0)
{
throw new Exception("Skip count must be greater than zero");
}
CurrentSource.SkipRecords = option.ParsedAsInt;
ReadNextToken = true;
break;
}
case "DELIMITER":
case "DELIM":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
{
throw new Exception(String.Format("Expected a valid delimiter after {0}", token.Data));
}
CurrentSource.Delimiter = HqlTokenProcessor.CleanupDelimiter(option.Data);
ReadNextToken = true;
break;
}
case "AS":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.UNKNOWN)
{
throw new Exception("Expected a table reference name following AS");
}
CurrentSource.TableReference = option.Data;
break;
}
default:
breakOut = true;
break;
}
}
else
{
breakOut = true;
}
if (breakOut)
{
ReadNextToken = false;
break;
}
}
if (joinType != HqlKeyword.UNKNOWN)
{
HqlDataSource src = LeftSideOfJoin;
src.JoinType = joinType;
if (ReadNextToken)
{
processor.MoveNextToken();
token = processor.GetToken();
}
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.ON)
{
throw new Exception("Expected ON following a table");
}
src.InitJoining();
src.OnJoin.Parse(processor, HqlKeyword.ON, HqlKeyword.ON, null);
ReadNextToken = false;
}
}
if (FileExpected)
{
throw new Exception("Expected additional FROM fields");
}
}
private void AddTextReader(HqlStream stream)
{
HqlDataSource source = new HqlDataSource(HqlDataSourceType.HQLQUERY, stream);
AddSource(source);
}
private void AddDual()
{
HqlDataSource source = new HqlDataSource(HqlDataSourceType.DUAL, String.Empty);
AddSource(source);
}
private void AddStdin()
{
HqlDataSource source = new HqlDataSource(HqlDataSourceType.STDIN, String.Empty);
AddSource(source);
}
///////////////////////
// Private
private void AddFile(string name)
{
HqlDataSource source = new HqlDataSource(HqlDataSourceType.FILE, name);
AddSource(source);
}
//public HqlValues MergeValues(HqlValues priorvalues, HqlValues thisvalues, ref HqlFieldGroup combinedFields)
//{
// if (combinedFields == null)
// combinedFields = new HqlFieldGroup(priorvalues.FieldsImpacted, thisvalues.FieldsImpacted);
// return new HqlValues(combinedFields, priorvalues, thisvalues);
//}
public bool ReadThroughUsingJoin(HqlDataSource prior, HqlWhere onJoin, HqlWhere where, bool selectRecord)
{
// TODO
// - need to sort the prior how I'm going to join these.
// - read through current and join them and save to new object
// - clear all the saved data from the prior object
//HqlFieldGroup combinedFieldsImpacted = null;
//HqlValuesComparer comparer = prior.comparer;
// TODO, sort the prior
//bool sortedPrior = false;
for (; ;)
{
if (selectRecord && Lines.Count > 0)
{
_resultValues = Lines[0];
Lines.RemoveAt(0);
if (where.Evaluate(_resultValues))
{
return(true);
}
continue;
}
if (EndOfStream)
{
if (!_completedJoin)
{
_completedJoin = true;
if (prior != null && (prior.JoinType == HqlKeyword.LEFT_OUTER || prior.JoinType == HqlKeyword.FULL_OUTER))
{
prior.Comparer.JoinMethod = HqlJoinMethod.RETURN_ALL;
prior.Comparer.Reset();
prior.comparer.SetValues(BlankValues);
for (; prior.Comparer.MoveNext();)
{
prior.Comparer.Current.IncreaseUsedCount();
HqlValues combinedvalues = prior.Comparer.CombinedValues;
SaveRowValue(combinedvalues);
}
continue;
}
}
_resultValues = null;
return(false);
}
//string fromline = ReadLine();
//if (fromline.Equals(String.Empty) && EndOfStream)
// continue;
//HqlLine line = new HqlLine(fromline, settings, RowNum)
HqlLine line = ReadLine();
if (line.IsBlank && EndOfStream)
{
continue;
}
#if DEBUG
if (line.Line.StartsWith("1503|706"))
{
int stop = 0;
}
#endif
// get the current line of data
HqlValues thisvalues = new HqlValues(FieldsImpacted, line);
// does this match the where statement?
if (JoinType == HqlKeyword.LEFT_OUTER || JoinType == HqlKeyword.FULL_OUTER)
{
// pass through, don't evaluate the ON command because I want this regardless
}
else if (prior != null && (prior.JoinType == HqlKeyword.FULL_OUTER || prior.JoinType == HqlKeyword.RIGHT_OUTER))
{
// pass through
}
else if (!onJoin.EvaluateJoin(thisvalues, true)) // || !where.EvaluateJoin(thisvalues))
{
// didn't pass through, so continue
continue;
}
// This is here for the first "join"
// I need to save all the records.
if (prior == null)
{
SaveRowValue(thisvalues);
continue;
}
// now time to join the data!
bool savedThisRecord = false;
// do preprocessing if this is to be searched through sorting
if (prior.Comparer == null)
{
if (prior.Lines.Count == 0)
{
throw new Exception("Data integrity issue. Didn't find anything to join with on prior! TODO, this can be valid, need to fix!!");
}
HqlValues priorvalues = prior.Lines[0]; // guaranteed to be there
prior.Comparer = new HqlValuesComparer(prior, onJoin, where); //, settings);
prior.Comparer.CombinedFieldsImpacted = new HqlFieldGroup(priorvalues.FieldsImpacted, thisvalues.FieldsImpacted);
if (prior.Comparer.CountOfJoinFields == 0)
{
prior.Comparer.JoinMethod = HqlJoinMethod.LINEAR_COMPARE;
}
Console.Error.WriteLine(String.Format("Using joining of {0} between ({1}) {2} and ({3}) {4}",
prior.Comparer.JoinMethod.ToString(),
prior.TableReference,
prior.SourceName,
this.TableReference,
this.SourceName)
);
if (prior.Comparer.JoinMethod == HqlJoinMethod.SORT_COMPARE)
{
prior.Comparer.SortPrior();
}
}
prior.Comparer.SetValues(thisvalues);
prior.Comparer.Reset();
for (; prior.Comparer.MoveNext();)
{
prior.Comparer.Current.IncreaseUsedCount();
HqlValues combinedvalues = prior.Comparer.CombinedValues;
SaveRowValue(combinedvalues);
savedThisRecord = true;
}
if (!savedThisRecord && (prior.JoinType == HqlKeyword.RIGHT_OUTER || prior.JoinType == HqlKeyword.FULL_OUTER))
{
HqlValues combinedvalues = new HqlValues(prior.Comparer.CombinedFieldsImpacted, prior.BlankValues, thisvalues);
SaveRowValue(combinedvalues);
}
}
}
