public void Parse(HqlTokenProcessor processor, HqlKeyword firstToken, HqlKeyword thisKeyword, HqlClause select)
{
HqlToken token;
int openParen = 0;
// first thing should be WHERE
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != firstToken)
{
return;
}
// now pull back things until you get to EOF or GROUP
processor.MoveNextToken();
for (int i = 0; ; i++)
{
if (!ParseCompare(processor, (i > 0), ref openParen, true, thisKeyword, select))
{
break;
}
}
if (openParen != 0)
{
throw new Exception("Unbalanced parens");
}
}
///////////////////////
// Private
private void ProcessOutFilename(HqlTokenProcessor processor)
{
if (OutputFilename == null || OutputFilename.Length == 0)
{
throw new Exception("Output filename cannot be empty");
}
if (OutputFilename.Contains("{"))
{
_output = new HqlOutput(OutputFilename, this);
int currloc = 0;
int startloc = 0;
for (; ;)
{
bool FoundEndBrace = false;
currloc = OutputFilename.IndexOf('{', startloc);
if (currloc < 0 || currloc >= OutputFilename.Length)
{
break;
}
startloc = currloc;
string innerField = String.Empty;
for (currloc++; currloc < OutputFilename.Length; currloc++)
{
if (OutputFilename[currloc] == '}')
{
// this means that in the string was {} because there is no gap between them for characters
if (startloc + 1 == currloc)
{
throw new Exception(String.Format("Cannot have an empty field in OUTPUT name of |{0}|", OutputFilename));
}
startloc = currloc;
FoundEndBrace = true;
break;
}
innerField += OutputFilename[currloc];
}
if (!FoundEndBrace)
{
throw new Exception(String.Format("Need a closing brace in filename |{0}|", OutputFilename));
}
HqlTokenProcessor newProcessor = new HqlTokenProcessor(innerField, processor);
newProcessor.MoveNextToken();
HqlToken token = newProcessor.GetToken();
if (token.WordType == HqlWordType.UNKNOWN)
{
newProcessor.CheckForTokenReference(ref token);
}
if (token.WordType != HqlWordType.FIELD && token.WordType != HqlWordType.SCALAR)
{
throw new Exception("Info in braces is not a field");
}
token.Field.FieldRename = innerField; // set the original name of the field for the {replace}
_output.FieldGroup.AddField(token.Field);
}
}
}
///////////////////////
// Overridden functions
///////////////////////
// Public
public void Parse(HqlTokenProcessor processor)
{
HqlToken token;
HqlToken option;
// first thing should be from
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.WITH)
{
return; // throw new Exception("Expected WITH");
}
// now pull back things until you get to EOF
for (; ;)
{
processor.MoveNextToken();
token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
break;
}
if (
token.WordType == HqlWordType.UNKNOWN
||
token.WordType == HqlWordType.KEYWORD
)
{
switch (token.Data.ToUpper())
{
case "HEADER":
{
PrintHeader = true;
break;
}
case "PQ":
case "PRESERVE-QUOTES":
case "PRESERVE_QUOTES":
{
// TODO, not implemented
PreserveQuotes = true;
break;
}
case "TEMPDIR":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
{
throw new Exception(String.Format("Expected a valid directory after {0}", token.Data));
}
// TODO, save directory
break;
}
case "OD":
case "OUT-DELIMITER":
case "OUT_DELIMITER":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
{
throw new Exception(String.Format("Expected a valid delimiter after {0}", token.Data));
}
OutDelimiter = HqlTokenProcessor.CleanupDelimiter(option.Data);
break;
}
case "D":
case "DELIM":
case "DELIMITER":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
{
throw new Exception(String.Format("Expected a valid delimiter after {0}", token.Data));
}
InDelimiter = HqlTokenProcessor.CleanupDelimiter(option.Data);
break;
}
case "PFD":
case "PRINT-FINAL-DELIMITER":
case "PRINT_FINAL_DELIMITER":
{
HasFinalDelimiter = true;
break;
}
case "OUTPUT":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
{
throw new Exception(String.Format("Expected a filename after {0}", token.Data));
}
OutputFilename = option.Data;
ProcessOutFilename(processor);
break;
}
//case "SKIP":
// {
// option = GetOptionData(processor, token.Data);
// if (option.WordType != HqlWordType.INT || (int)option.Parsed <= 0)
// throw new Exception(String.Format("Expected a greater than zero integer after {0}", token.Data));
// SkipRecords = (Int32)(Int64)option.Parsed;
// break;
// }
default:
throw new Exception(String.Format("Unknown WITH option of {0}", token.Data.ToString()));
}
}
else
{
throw new Exception("Unknown type in WITH clause");
}
}
}
///////////////////////
// Overridden functions
///////////////////////
// Public
public override void Parse(HqlTokenProcessor processor)
{
HqlToken token;
bool FieldExpected = true;
// first thing should be orderby
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.ORDERBY)
{
return;
}
// now pull back things until you get to HAVING or EOL
for (; ;)
{
processor.MoveNextToken();
token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
return;
}
if (token.WordType == HqlWordType.KEYWORD && (token.Keyword == HqlKeyword.ASCENDING || token.Keyword == HqlKeyword.DESCENDING))
{
_fieldgroup.SetPreviousDirection(token.Keyword);
continue;
}
if (token.WordType == HqlWordType.KEYWORD &&
(
token.Keyword == HqlKeyword.WITH ||
token.Keyword == HqlKeyword.HAVING
)
)
{
break;
}
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.COMMA)
{
FieldExpected = true;
continue;
}
// else time to process it!
if (!FieldExpected)
{
throw new Exception("Found additional ORDER-BY fields not expected");
}
FieldExpected = false;
// * = everything
// FIELD = field<NUM> such as field1, field2, field10
// FIXEDWIDTH = NAME(S, L) | NAME(S,L) such as bob(1, 5) or susan(10,5)
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.STAR)
{
throw new Exception("Cannot ORDER-BY STAR");
}
else if (token.WordType == HqlWordType.FIELD)
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.SCALAR)
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.ROWNUM)
{
AddField(token.Field);
}
else
{
throw new Exception("Cannot determine data in ORDER-BY clause");
}
}
if (FieldExpected)
{
throw new Exception("Expected additional GROUPBY fields");
}
}
public void Parse(HqlTokenProcessor processor)
{
HqlToken token;
bool FileExpected = true;
HqlKeyword joinType;
// first thing should be from
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.FROM)
{
throw new Exception("Expected FROM");
}
bool ReadNextToken = true;
// now pull back things until you get to EOF or WHERE
for (; ;)
{
joinType = HqlKeyword.UNKNOWN;
if (ReadNextToken)
{
processor.MoveNextToken();
}
ReadNextToken = true;
token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
break;
}
if (token.WordType == HqlWordType.KEYWORD &&
(
token.Keyword == HqlKeyword.WHERE ||
token.Keyword == HqlKeyword.GROUPBY ||
token.Keyword == HqlKeyword.HAVING ||
token.Keyword == HqlKeyword.ORDERBY
)
)
{
break;
}
if (token.WordType == HqlWordType.KEYWORD &&
(
token.Keyword == HqlKeyword.FULL_OUTER ||
token.Keyword == HqlKeyword.LEFT_OUTER ||
token.Keyword == HqlKeyword.RIGHT_OUTER ||
token.Keyword == HqlKeyword.INNER
)
)
{
joinType = token.Keyword;
processor.MoveNextToken();
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.JOIN)
{
throw new Exception("Expected JOIN keyword");
}
FileExpected = true;
processor.MoveNextToken();
token = processor.GetToken();
}
// else time to process it!
if (!FileExpected)
{
break;
}
FileExpected = false;
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.STDIN)
{
AddStdin();
}
else if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.DUAL)
{
AddDual();
}
else if (token.WordType == HqlWordType.TEXT || token.WordType == HqlWordType.LITERAL_STRING)
{
AddFile(token.Data);
}
else if (token.WordType == HqlWordType.STREAM)
{
HqlDataSource source = new HqlDataSource(HqlDataSourceType.STREAM, (System.IO.StreamReader)token.Parsed);
AddSource(source);
}
else if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.OPENPAREN)
{
// NESTED QUERY!
HqlStream cs = new HqlStream(processor.GetRemainingSql(false), ")");
string remainingSql = cs.RemainingSql;
processor.SetRemainingSql(remainingSql);
processor.MoveNextToken();
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.CLOSEDPAREN)
{
throw new Exception("Nested query SQL did not return correctly");
}
AddTextReader(cs);
}
else
{
throw new Exception("Unknown type of FROM clause");
}
// look to process SKIP
for (;;)
{
bool breakOut = false;
if (ReadNextToken)
{
processor.MoveNextToken();
token = processor.GetToken();
}
if (_sources != null && _sources.Length > 0 && (token.WordType == HqlWordType.UNKNOWN || token.WordType == HqlWordType.KEYWORD))
{
HqlToken option;
switch (token.Data.ToUpper())
{
case "SKIP":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.INT || option.ParsedAsInt <= 0)
{
throw new Exception("Skip count must be greater than zero");
}
CurrentSource.SkipRecords = option.ParsedAsInt;
ReadNextToken = true;
break;
}
case "DELIMITER":
case "DELIM":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.TEXT && option.WordType != HqlWordType.LITERAL_STRING)
{
throw new Exception(String.Format("Expected a valid delimiter after {0}", token.Data));
}
CurrentSource.Delimiter = HqlTokenProcessor.CleanupDelimiter(option.Data);
ReadNextToken = true;
break;
}
case "AS":
{
option = processor.GetOptionData(token.Data);
if (option.WordType != HqlWordType.UNKNOWN)
{
throw new Exception("Expected a table reference name following AS");
}
CurrentSource.TableReference = option.Data;
break;
}
default:
breakOut = true;
break;
}
}
else
{
breakOut = true;
}
if (breakOut)
{
ReadNextToken = false;
break;
}
}
if (joinType != HqlKeyword.UNKNOWN)
{
HqlDataSource src = LeftSideOfJoin;
src.JoinType = joinType;
if (ReadNextToken)
{
processor.MoveNextToken();
token = processor.GetToken();
}
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.ON)
{
throw new Exception("Expected ON following a table");
}
src.InitJoining();
src.OnJoin.Parse(processor, HqlKeyword.ON, HqlKeyword.ON, null);
ReadNextToken = false;
}
}
if (FileExpected)
{
throw new Exception("Expected additional FROM fields");
}
}
///////////////////////
// Private
protected bool ParseCompare(HqlTokenProcessor processor, bool andorAllowed, ref int openParen, bool closedAllowed, HqlKeyword thisKeyword, HqlClause select)
{
#if DEBUG
if (thisKeyword != HqlKeyword.WHERE && thisKeyword != HqlKeyword.HAVING && thisKeyword != HqlKeyword.ON)
{
throw new Exception("Cannot access ParseCompare with invalid keyword");
}
#endif
HqlToken token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
return(false);
}
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.CLOSEDPAREN)
{
if (!closedAllowed)
{
throw new Exception("Empty Parens found");
}
openParen--;
return(false);
}
if (token.WordType == HqlWordType.KEYWORD &&
(
(
thisKeyword == HqlKeyword.ON &&
(
token.Keyword == HqlKeyword.WHERE ||
token.Keyword == HqlKeyword.INNER ||
token.Keyword == HqlKeyword.RIGHT_OUTER ||
token.Keyword == HqlKeyword.LEFT_OUTER ||
token.Keyword == HqlKeyword.FULL_OUTER ||
token.Keyword == HqlKeyword.WHERE ||
token.Keyword == HqlKeyword.GROUPBY ||
token.Keyword == HqlKeyword.HAVING ||
token.Keyword == HqlKeyword.ORDERBY ||
token.Keyword == HqlKeyword.WITH
)
)
||
(
thisKeyword == HqlKeyword.WHERE &&
(
token.Keyword == HqlKeyword.INNER ||
token.Keyword == HqlKeyword.RIGHT_OUTER ||
token.Keyword == HqlKeyword.LEFT_OUTER ||
token.Keyword == HqlKeyword.FULL_OUTER ||
token.Keyword == HqlKeyword.WHERE ||
token.Keyword == HqlKeyword.GROUPBY ||
token.Keyword == HqlKeyword.HAVING ||
token.Keyword == HqlKeyword.ORDERBY ||
token.Keyword == HqlKeyword.WITH
)
)
||
(
thisKeyword == HqlKeyword.HAVING &&
(
token.Keyword == HqlKeyword.ORDERBY ||
token.Keyword == HqlKeyword.WITH
)
)
)
)
{
return(false);
}
if (token.WordType == HqlWordType.KEYWORD && (token.Keyword == HqlKeyword.AND || token.Keyword == HqlKeyword.OR))
{
if (!andorAllowed)
{
throw new Exception("Unexpected AND/OR");
}
_q.Add(token);
processor.MoveNextToken();
return(true);
}
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.OPENPAREN)
{
openParen++;
_q.Add(token);
processor.MoveNextToken();
for (int i = 0; ; i++)
{
if (!ParseCompare(processor, (i > 0), ref openParen, (i > 0), thisKeyword, select))
{
break;
}
}
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.CLOSEDPAREN)
{
throw new Exception(String.Format("Unbalanced Parens in {0}", _name));
}
_q.Add(token);
processor.MoveNextToken();
}
else
{
HqlToken token1 = token;
if (token1.WordType == HqlWordType.UNKNOWN && !processor.CheckForTokenReference(ref token1))
{
throw new Exception(String.Format("Unknown {0} reference to {1}", _name, token1.Data));
}
// HACK - sort of a hack this says if I can't find this field
// then add it to the select but make it not print out
HqlEvaluator.VerifyFieldPresence(select, token1);
// END HACK
processor.MoveNextToken();
HqlToken compare = processor.GetToken();
// this could be
// - a COMPARE such as |field > 0|
// - an IN such as |field in ('A', 'B')|
// - an IN such as |field in (select item from bob)|
if (compare.WordType == HqlWordType.KEYWORD && compare.Keyword == HqlKeyword.COMPARE)
{
processor.MoveNextToken();
HqlToken token2 = processor.GetToken();
if (token2.WordType == HqlWordType.UNKNOWN && !processor.CheckForTokenReference(ref token2))
{
throw new Exception(String.Format("Unknown WHERE reference to {0}", token2.Data));
}
// HACK - sort of a hack this says if I can't find this field
// then add it to the select but make it not print out
HqlEvaluator.VerifyFieldPresence(select, token2);
// END HACK
HqlCompareToken comparetoken = new HqlCompareToken(token1, compare, token2);
_q.Add(comparetoken);
processor.MoveNextToken();
}
else if (compare.WordType == HqlWordType.KEYWORD && (compare.Keyword == HqlKeyword.IN || compare.Keyword == HqlKeyword.NOT_IN))
{
// now need to check for values or select
processor.MoveNextToken();
HqlToken paren = processor.GetToken();
if (paren.WordType != HqlWordType.KEYWORD || paren.Keyword != HqlKeyword.OPENPAREN)
{
throw new Exception(String.Format("Expected an open paren after IN in {0}", _name));
}
ArrayList compareValues = new ArrayList();
bool ExpectedValues = true;
for (int count = 0; ; count++)
{
processor.MoveNextToken();
HqlToken val = processor.GetToken();
if (val.WordType == HqlWordType.KEYWORD && val.Keyword == HqlKeyword.CLOSEDPAREN)
{
break;
}
if (val.WordType == HqlWordType.KEYWORD && val.Keyword == HqlKeyword.COMMA)
{
ExpectedValues = true;
continue;
}
if (val.WordType == HqlWordType.KEYWORD && val.Keyword == HqlKeyword.SELECT)
{
ExpectedValues = false;
if (count != 0)
{
throw new Exception("You can only put a nested query in an IN statement if it is the only value present");
}
HqlTokenProcessor.GetResultOfNestedQueryForInStatement(processor, compareValues);
}
else if (
val.WordType == HqlWordType.TEXT ||
val.WordType == HqlWordType.INT ||
val.WordType == HqlWordType.FLOAT ||
val.WordType == HqlWordType.LITERAL_STRING ||
val.WordType == HqlWordType.FIELD ||
val.WordType == HqlWordType.SCALAR
)
{
ExpectedValues = false;
compareValues.Add(val);
// HACK - sort of a hack this says if I can't find this field
// then add it to the select but make it not print out
HqlEvaluator.VerifyFieldPresence(select, val);
// END HACK
}
else
{
throw new Exception("Found invalid value in IN clause");
}
}
if (ExpectedValues)
{
throw new Exception("Expected additional values in IN clause");
}
if (compareValues.Count == 0)
{
throw new Exception("No values present in IN clause");
}
HqlCompareToken comparetoken = new HqlCompareToken(token1, compare, compareValues);
_q.Add(comparetoken);
processor.MoveNextToken();
}
else
{
throw new Exception(String.Format("Unknown {0} compare", _name));
}
}
return(true);
}
///////////////////////
// Overridden functions
///////////////////////
// Public
public override void Parse(HqlTokenProcessor processor)
{
HqlToken token;
bool FieldExpected = true;
// first thing should be groupby
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.GROUPBY)
{
return;
}
// now pull back things until you get to HAVING or ORDER BY or EOL
for (; ;)
{
processor.MoveNextToken();
token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
return;
}
if (token.WordType == HqlWordType.KEYWORD &&
(
token.Keyword == HqlKeyword.HAVING ||
token.Keyword == HqlKeyword.ORDERBY ||
token.Keyword == HqlKeyword.WITH
)
)
{
break;
}
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.COMMA)
{
FieldExpected = true;
continue;
}
// else time to process it!
if (!FieldExpected)
{
throw new Exception("Found additional GROUPBY fields not expected");
}
FieldExpected = false;
// * = everything
// FIELD = field<NUM> such as field1, field2, field10
// FIXEDWIDTH = NAME(S, L) | NAME(S,L) such as bob(1, 5) or susan(10,5)
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.STAR)
{
throw new Exception("Cannot GROUPBY STAR");
}
else if (token.WordType == HqlWordType.FIELD)
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.SCALAR)
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.LITERAL_STRING)
{
HqlField f = new HqlField(HqlFieldType.LITERAL_STRING, token.Data);
AddField(f);
}
else if (token.WordType == HqlWordType.UNKNOWN && processor.CheckForTokenReference(ref token))
{
AddField(token.Field);
}
else
{
throw new Exception("Cannot determine data in GROUPBY clause");
}
}
if (FieldExpected)
{
throw new Exception("Expected additional GROUPBY fields");
}
}
///////////////////////
// Overridden functions
///////////////////////
// Public
public override void Parse(HqlTokenProcessor processor)
{
HqlToken token;
bool FieldExpected = true;
// first thing should be select
token = processor.GetToken();
if (token.WordType != HqlWordType.KEYWORD || token.Keyword != HqlKeyword.SELECT)
{
throw new Exception("Expected SELECT");
}
// now pull back things until you get to FROM
for (; ;)
{
processor.MoveNextToken();
token = processor.GetToken();
if (processor.MatchesEndOfProcessing())
{
throw new Exception("Reached EOL in SELECT clause");
}
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.FROM)
{
break;
}
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.COMMA)
{
FieldExpected = true;
continue;
}
if (_fieldgroup != null && _fieldgroup.Count > 0 && token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.AS)
{
processor.MoveNextToken();
token = processor.GetToken();
if (token.WordType != HqlWordType.TEXT && token.WordType != HqlWordType.UNKNOWN)
{
throw new Exception("Expected a table reference name following AS");
}
_fieldgroup.SetPreviousFieldRename(token.Data);
continue;
}
// else time to process it!
if (!FieldExpected)
{
throw new Exception("Found additional SELECT fields not expected");
}
FieldExpected = false;
if (token.WordType == HqlWordType.KEYWORD && token.Keyword == HqlKeyword.STAR)
{
AddField(new HqlField(HqlFieldType.STAR));
}
else if (token.WordType == HqlWordType.FIELD)
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.SCALAR)
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.LITERAL_STRING)
{
HqlField f = new HqlField(HqlFieldType.LITERAL_STRING, token.Data);
AddField(f);
}
else if (token.WordType == HqlWordType.UNKNOWN && processor.CheckForTokenReference(ref token))
{
AddField(token.Field);
}
else if (token.WordType == HqlWordType.ROWNUM)
{
AddField(token.Field);
}
else
{
throw new Exception("Cannot determine data in SELECT clause");
}
}
if (FieldExpected)
{
throw new Exception("Expected additional SELECT fields");
}
}