public void TestReadNumber()
{
string test1 = "this is a 123.5 notAnumber";
TextReader reader = new StringReader(test1);
StreamTokenizer tokenizer = new StreamTokenizer(reader, true);
TokenType tokentype1 = tokenizer.NextToken(); //this
TokenType tokentype2 = tokenizer.NextToken(); //is
TokenType tokentype3 = tokenizer.NextToken(); //a
tokenizer.NextToken();
double number = tokenizer.GetNumericValue();
Assertion.AssertEquals("test1", 123.5, number);
tokenizer.NextToken();
try
{
double number2 = tokenizer.GetNumericValue();
Assertion.Fail("This should fail because the token is not a number.");
}
catch
{
}
}
static void Main(string[] args)
{
// DEMO BOILERPLATE:
// create a test stream, with some sample text in it
MemoryStream ms = new MemoryStream();
StreamWriter sw = new StreamWriter(ms);
sw.WriteLine("This is a sample stream");
sw.WriteLine("with\tsome # this is a comment");
sw.WriteLine(" (example) text // this is another comment");
sw.WriteLine("and several /comments");
sw.Flush();
ms.Seek(0, SeekOrigin.Begin);
// and open a reader on it:
StreamReader sr = new StreamReader(ms);
// could also just open a file:
//StreamReader sr = new StreamReader("inputfile.txt");
// DEMO START:
StreamTokenizer tok = new StreamTokenizer(sr,
null, // whitespace delimiters
new char [] { '#', '/' }); // comment start chars are tokens too
string s;
while ((s = tok.NextToken()) != null)
{
Console.WriteLine("line {0} token: '{1}'", tok.Linenum, s);
if (s == "#")
{
Console.WriteLine(" (# comment, skipping to end of line)");
tok.SkipToEOL();
}
if (s == "/")
{
// peek at the next token to see if it is our
// second slash:
s = tok.NextToken();
if (s == null)
{
break;
}
if (s == "/")
{
Console.WriteLine(" (// comment, skipping)");
tok.SkipToEOL();
}
else
{
// no, so push the token back into the tokenizer
tok.Unget(s);
}
}
}
Console.WriteLine("[Hit return to exit]");
Console.ReadLine();
}
private bool ParseSentencesFromKIFFile()
{
using (var sr = new StreamReader(kifFilePath))
{
try
{
var st = new StreamTokenizer(sr)
{
EOLIsSignificant = true
};
Token nextToken = null;
int parenCount;
while ((nextToken = st.NextToken()).TheTokenType != TokenType.EndOfFile)
{
var input = new List <Token>();
parenCount = 0;
while (nextToken.TheTokenType != TokenType.EndOfFile)
{
input.Add(nextToken);
if (nextToken.TheTokenType == TokenType.LeftParen)
{
parenCount++;
}
else if (nextToken.TheTokenType == TokenType.RightParen)
{
if (--parenCount == 0)
{
goto EndOfSentence;
}
}
nextToken = st.NextToken();
}
EndOfSentence:
sentences.Add(input);
}
}
catch (Exception ex)
{
throw;
}
}
return(true);
}
private double GetNextNumber(StreamTokenizer tokenizer)
{
int type = tokenizer.NextToken();
if (type == StreamTokenizer.TT_WORD)
{
if (tokenizer.StringValue.Equals(NAN_SYMBOL))
{
return(Double.NaN);
}
else
{
try
{
NumberFormatInfo NFI = new NumberFormatInfo()
{
NumberDecimalSeparator = "."
};
return(Double.Parse(tokenizer.StringValue, NFI));
}
catch (FormatException ex)
{
throw new WKTParseException(ex);
}
}
}
throw new WKTParseException("Excepted: number");
}
private static String GetNextWord(StreamTokenizer tokenizer)
{
try
{
int type = tokenizer.NextToken();
switch (type)
{
case StreamTokenizer.TT_WORD:
String word = tokenizer.StringValue;
if (word.Equals(EMPTY))
{
return(EMPTY);
}
return(word);
case '(':
return(L_PAREN);
case ')':
return(R_PAREN);
case ',':
return(COMMA);
}
throw new WKTParseException($"Uknown type: '{( char )type}'");
}
catch (IOException ex)
{
throw new WKTParseException(ex);
}
}
/// <summary>
/// Gets the next word from the tokenizer
/// @throws StreamCorruptedException if the word does not match
/// @throws IOException if an error occurs while loading the data
/// </summary>
/// <returns>the next word</returns>
public string GetString()
{
if (_tokenizer == null)
{
return(null);
}
if (_putbackList.Count != 0)
{
var retVal = _putbackList[_putbackList.Count - 1];
_putbackList.RemoveAt(_putbackList.Count - 1);
return(retVal);
}
_tokenizer.NextToken();
if (_tokenizer.Ttype == StreamTokenizer.TtEOF)
{
_atEof = true;
}
if (_tokenizer.Ttype != StreamTokenizer.TtWord && _tokenizer.Ttype != StreamTokenizer.TtEol &&
_tokenizer.Ttype != StreamTokenizer.TtEOF)
{
Corrupt("word expected but not found");
}
if (_tokenizer.Ttype == StreamTokenizer.TtEol || _tokenizer.Ttype == StreamTokenizer.TtEOF)
{
return(null);
}
return(_tokenizer.StringValue);
}
private static char Unescape(String text)
{
StreamTokenizer parser = new StreamTokenizer(new StringReader(text));
parser.NextToken();
return(parser.StringValue[0]);
}
public void TestTokenize5()
{
string test1 = "-2.5 -2 4";
TextReader reader = new StringReader(test1);
StreamTokenizer tokenizer = new StreamTokenizer(reader, true);
TokenType tokentype = tokenizer.NextToken();
int iTokenCount = 0;
while (tokentype != TokenType.Eof)
{
tokentype = tokenizer.NextToken();
iTokenCount++;
}
// first token will be -2.5
Assertion.AssertEquals("token count", 3, iTokenCount);
}
public void TestTokenize3()
{
string test1 = "this is a very_long_word and long123 long123longer ok";
TextReader reader = new StringReader(test1);
StreamTokenizer tokenizer = new StreamTokenizer(reader, true);
TokenType tokentype = tokenizer.NextToken();
int iTokenCount = 0;
while (tokentype != TokenType.Eof)
{
//Console.WriteLine("token:"+tokentype+"("+tokenizer.GetStringValue()+")");
tokentype = tokenizer.NextToken();
iTokenCount++;
}
//Console.WriteLine("token count ignore=false="+iTokenCount);
Assertion.AssertEquals("token count", 8, iTokenCount);
}
public void TestTokenize2()
{
string test1 = "this,.is 123 test 456.789 test123 123.2 12*2 /* hello */ \n hello";
TextReader reader = new StringReader(test1);
StreamTokenizer tokenizer = new StreamTokenizer(reader, false);
TokenType tokentype = tokenizer.NextToken();
int iTokenCount = 0;
while (tokentype != TokenType.Eof)
{
//Console.WriteLine("token:"+tokentype+"("+tokenizer.GetStringValue()+")");
tokentype = tokenizer.NextToken();
iTokenCount++;
}
//Console.WriteLine("token count ignore=false="+iTokenCount);
Assertion.AssertEquals("token count", 30, iTokenCount);
}
private static bool HasNext(StreamTokenizer st)
{
if (st == null)
{
throw new ArgumentNullException(nameof(st));
}
return(st.NextToken() != StreamTokenizer.TtEof);
}
public void NumericTokenTest()
{
//Set up tokenizer
Tokenizer = new StreamTokenizer(new StringReader("100"));
Tokenizer.NextToken();
//Test token
Assert.AreEqual(100, Tokenizer.GetNumericValue());
}
public void NumericTokenTest()
{
//Set up tokenizer
Tokenizer = new StreamTokenizer(new StringReader("100"));
Tokenizer.NextToken();
//Test token
Assert.AreEqual(100, Tokenizer.GetNumericValue());
}
/// <exception cref="System.IO.IOException"/>
internal virtual Parser.Token Next()
{
int type = tok.NextToken();
switch (type)
{
case StreamTokenizer.TtEof:
case StreamTokenizer.TtEol:
{
return(null);
}
case StreamTokenizer.TtNumber:
{
return(new Parser.NumToken(tok.nval));
}
case StreamTokenizer.TtWord:
{
return(new Parser.StrToken(Parser.TType.Ident, tok.sval));
}
case '"':
{
return(new Parser.StrToken(Parser.TType.Quot, tok.sval));
}
default:
{
switch (type)
{
case ',':
{
return(new Parser.Token(Parser.TType.Comma));
}
case '(':
{
return(new Parser.Token(Parser.TType.Lparen));
}
case ')':
{
return(new Parser.Token(Parser.TType.Rparen));
}
default:
{
throw new IOException("Unexpected: " + type);
}
}
break;
}
}
}
/// <summary>
/// Reads partial atomic charges and add the to the given ChemModel.
/// </summary>
/// <param name="model"></param>
private void ReadPartialCharges(IChemModel model)
{
Trace.TraceInformation("Reading partial atomic charges");
var moleculeSet = model.MoleculeSet;
IAtomContainer molecule = moleculeSet[0];
string line = input.ReadLine();
// skip first line after "Total atomic charges"
while (true)
{
line = input.ReadLine();
Debug.WriteLine($"Read charge block line: {line}");
if ((line == null) || line.Contains("Sum of Mulliken charges"))
{
Debug.WriteLine("End of charge block found");
break;
}
StringReader sr = new StringReader(line);
StreamTokenizer tokenizer = new StreamTokenizer(sr);
if (tokenizer.NextToken() == StreamTokenizer.TTypeNumber)
{
int atomCounter = (int)tokenizer.NumberValue;
tokenizer.NextToken();
// ignore the symbol
double charge;
if (tokenizer.NextToken() == StreamTokenizer.TTypeNumber)
{
charge = tokenizer.NumberValue;
Debug.WriteLine("Found charge for atom " + atomCounter + ": " + charge);
}
else
{
throw new CDKException("Error while reading charge: expected double.");
}
IAtom atom = molecule.Atoms[atomCounter - 1];
atom.Charge = charge;
}
}
}
/// <summary>
/// Returns the next word in the stream as uppercase text.
/// </summary>
/// <param name="tokenizer">
/// Tokenizer over a stream of text in Well-known Text
/// format. The next token must be a word.
/// </param>
/// <returns>
/// The next word in the stream as uppercase text.
/// </returns>
/// <exception cref="GeometryIOException">
/// If the next token is not a word.
/// </exception>
/// <exception cref="IOException">
/// If an I/O error occurs.
/// </exception>
private string GetNextWord(StreamTokenizer tokenizer)
{
Token token = null;
if (!tokenizer.NextToken(out token))
{
return(null);
}
TokenType type = token.Type;
switch (type)
{
case TokenType.Eof:
throw new GeometryIOException("Expected word but encountered end of stream");
case TokenType.Eol:
throw new GeometryIOException("Expected word but encountered end of line");
case TokenType.Float:
throw new GeometryIOException("Expected word but encountered number: " + token.StringValue);
case TokenType.Integer:
throw new GeometryIOException("Expected word but encountered number: " + token.StringValue);
case TokenType.Word:
return(token.StringValue.ToUpper(CultureInfo.InvariantCulture));
default:
{
string sVal = token.StringValue;
if (sVal == TokenLParan)
{
return(TokenLParan);
}
if (sVal == TokenRParan)
{
return(TokenRParan);
}
if (sVal == TokenComma)
{
return(TokenComma);
}
}
break;
}
Debug.Assert(false, "Should never reach here: Encountered unexpected StreamTokenizer type: " + type);
return(null);
}
/// <summary>
/// Set the params (analyzerName only), Comma-separate list of Analyzer class names. If the Analyzer lives in
/// Lucene.Net.Analysis, the name can be shortened by dropping the Lucene.Net.Analysis part of the Fully Qualified Class Name.
/// <para/>
/// Analyzer names may also refer to previously defined AnalyzerFactory's.
/// <para/>
/// Example Declaration:
/// <code>
/// {"NewAnalyzer" NewAnalyzer(WhitespaceAnalyzer, SimpleAnalyzer, StopAnalyzer, Standard.StandardAnalyzer) >
/// </code>
/// <para/>
/// Example AnalyzerFactory usage:
/// <code>
/// -AnalyzerFactory(name:'whitespace tokenized',WhitespaceTokenizer)
/// -NewAnalyzer('whitespace tokenized')
/// </code>
/// </summary>
/// <param name="params">analyzerClassName, or empty for the StandardAnalyzer</param>
public override void SetParams(string @params)
{
base.SetParams(@params);
StreamTokenizer stok = new StreamTokenizer(new StringReader(@params));
stok.QuoteChar('"');
stok.QuoteChar('\'');
stok.EndOfLineIsSignificant = false;
stok.OrdinaryChar(',');
try
{
while (stok.NextToken() != StreamTokenizer.TokenType_EndOfStream)
{
switch (stok.TokenType)
{
case ',':
{
// Do nothing
break;
}
case '\'':
case '\"':
case StreamTokenizer.TokenType_Word:
{
analyzerNames.Add(stok.StringValue);
break;
}
default:
{
throw RuntimeException.Create("Unexpected token: " + stok.ToString());
}
}
}
}
catch (Exception e) when(e.IsRuntimeException())
{
if (e.Message.StartsWith("Line #", StringComparison.Ordinal))
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
else
{
throw RuntimeException.Create("Line #" + (stok.LineNumber + AlgLineNum) + ": ", e);
}
}
catch (Exception t) when(t.IsThrowable())
{
throw RuntimeException.Create("Line #" + (stok.LineNumber + AlgLineNum) + ": ", t);
}
}
/// <summary>
/// Returns the next number in the stream.
/// </summary>
/// <param name="tokenizer">
/// Tokenizer over a stream of text in Well-known Text format.
/// The next token must be a number.
/// </param>
/// <returns>The next number in the stream.</returns>
/// <exception cref="GeometryIOException">
/// If the next token is not a number.
/// </exception>
/// <exception cref="IOException">
/// If an I/O error occurs.
/// </exception>
private double GetNextNumber(StreamTokenizer tokenizer)
{
Token token = null;
if (!tokenizer.NextToken(out token))
{
return(double.NaN);
}
TokenType type = token.Type;
switch (type)
{
case TokenType.Eof:
throw new GeometryIOException("Expected number but encountered end of stream");
case TokenType.Eol:
throw new GeometryIOException("Expected number but encountered end of line");
case TokenType.Float:
return(Convert.ToDouble(token.Object, m_objProvider));
case TokenType.Integer:
return(Convert.ToDouble(token.Object, m_objProvider));
case TokenType.Word:
throw new GeometryIOException("Expected number but encountered word: " + token.StringValue);
default:
{
string sVal = token.StringValue;
if (sVal == TokenLParan)
{
throw new GeometryIOException("Expected number but encountered '('");
}
if (sVal == TokenRParan)
{
throw new GeometryIOException("Expected number but encountered ')'");
}
if (sVal == TokenComma)
{
throw new GeometryIOException("Expected number but encountered ','");
}
}
break;
}
Debug.Assert(false, "Should never reach here: Encountered unexpected StreamTokenizer type: " + type);
return(0);
}
private static bool IsNumberText(StreamTokenizer tokenizer)
{
try
{
int type = tokenizer.NextToken();
tokenizer.PushBack();
return(type == StreamTokenizer.TT_WORD);
}
catch (IOException ex)
{
throw new WKTParseException(ex);
}
}
private static bool IsOpenerNext(StreamTokenizer tokenizer)
{
try
{
int type = tokenizer.NextToken();
tokenizer.PushBack();
return(type == '(');
}
catch (IOException ex)
{
throw new WKTParseException(ex);
}
}
/// <summary>
/// Set the params (analyzerName only), Comma-separate list of Analyzer class names. If the Analyzer lives in
/// Lucene.Net.Analysis, the name can be shortened by dropping the Lucene.Net.Analysis part of the Fully Qualified Class Name.
/// <para/>
/// Analyzer names may also refer to previously defined AnalyzerFactory's.
/// <para/>
/// Example Declaration:
/// <code>
/// {"NewAnalyzer" NewAnalyzer(WhitespaceAnalyzer, SimpleAnalyzer, StopAnalyzer, Standard.StandardAnalyzer) >
/// </code>
/// <para/>
/// Example AnalyzerFactory usage:
/// <code>
/// -AnalyzerFactory(name:'whitespace tokenized',WhitespaceTokenizer)
/// -NewAnalyzer('whitespace tokenized')
/// </code>
/// </summary>
/// <param name="params">analyzerClassName, or empty for the StandardAnalyzer</param>
public override void SetParams(string @params)
{
base.SetParams(@params);
StreamTokenizer stok = new StreamTokenizer(new StringReader(@params));
stok.QuoteChar('"');
stok.QuoteChar('\'');
stok.IsEOLSignificant = false;
stok.OrdinaryChar(',');
try
{
while (stok.NextToken() != StreamTokenizer.TT_EOF)
{
switch (stok.TokenType)
{
case ',':
{
// Do nothing
break;
}
case '\'':
case '\"':
case StreamTokenizer.TT_WORD:
{
analyzerNames.Add(stok.StringValue);
break;
}
default:
{
//throw new RuntimeException("Unexpected token: " + stok.ToString());
throw new Exception("Unexpected token: " + stok.ToString());
}
}
}
}
catch (Exception e)
{
if (e.Message.StartsWith("Line #", StringComparison.Ordinal))
{
throw e;
}
else
{
throw new Exception("Line #" + (stok.LineNumber + AlgLineNum) + ": ", e);
}
}
}
/// <summary>
/// Tests if reader is at EOF.
/// </summary>
private static bool IsAtEndOfFile(StreamReader bufferedReader)
{
var position = bufferedReader.BaseStream.Position;
var tokenizer = new StreamTokenizer(bufferedReader);
Token t;
if (!tokenizer.NextToken(out t) || t is EofToken)
{
return(true);
}
bufferedReader.BaseStream.Seek(position, SeekOrigin.Begin);
return(false);
}
private static void CheckValid(StreamTokenizer st, string token)
{
if (st == null)
{
throw new ArgumentNullException(nameof(st));
}
if (token == null)
{
throw new ArgumentNullException(nameof(token));
}
if (st.NextToken() != '[')
{
throw new IOException(string.Concat(token, " not followed by ["));
}
}
private bool IsNumberNext(StreamTokenizer tokenizer)
{
try
{
Token token = null;
if (!tokenizer.NextToken(out token))
{
return(false);
}
return(token.Type == TokenType.Float ||
token.Type == TokenType.Integer);
}
finally
{
tokenizer.PushBack();
}
}
public String readNextToken()
{
try
{
if (st.NextToken() == StreamTokenizer.TT_EOF)
{
return(null);
}
else
{
return(st.StringValue);
}
}
catch (Exception e)
{
string ex = e.ToString();
return(null);
}
}
private double[] GetCoordinate(StreamTokenizer tokenizer, bool tryParen)
{
bool opened;
if (opened = tryParen && IsOpenerNext(tokenizer))
{
tokenizer.NextToken();
}
double[] sequence = new double[2];
sequence[0] = GetNextNumber(tokenizer);
sequence[1] = GetNextNumber(tokenizer);
if (opened)
{
GetNextCloser(tokenizer);
}
return(sequence);
}
/// <summary>Internally fetches the next token.</summary>
/// <returns>the next token in the token stream, or null if none exists.</returns>
protected internal override string GetNext()
{
try
{
int nextTok = st.NextToken();
switch (nextTok)
{
case StreamTokenizer.TtEol:
{
return(eolString);
}
case StreamTokenizer.TtEof:
{
return(null);
}
case StreamTokenizer.TtWord:
{
return(st.sval);
}
case StreamTokenizer.TtNumber:
{
return(double.ToString(st.nval));
}
default:
{
char[] t = new char[] { (char)nextTok };
// (array initialization)
return(new string(t));
}
}
}
catch (IOException)
{
// do nothing, return null
return(null);
}
}
/// <summary>
/// Read algorithm from file.
/// Property examined: alt.tasks.packages == comma separated list of
/// alternate Assembly names where tasks would be searched for, when not found
/// in the default Assembly (that of <see cref="PerfTask"/>).
/// If the same task class appears in more than one Assembly, the Assembly
/// indicated first in this list will be used.
/// <para/>
/// The Lucene.Net implementation differs from Lucene in that all
/// referenced assemblies are also scanned for the type. However,
/// alt.tasks.packages may be included for assemblies that are
/// not referenced in your project.
/// </summary>
/// <param name="runData">perf-run-data used at running the tasks.</param>
/// <exception cref="Exception">if errors while parsing the algorithm.</exception>
public Algorithm(PerfRunData runData)
{
Config config = runData.Config;
taskPackages = InitTasksPackages(config);
string algTxt = config.AlgorithmText;
sequence = new TaskSequence(runData, null, null, false);
TaskSequence currSequence = sequence;
PerfTask prevTask = null;
StreamTokenizer stok = new StreamTokenizer(new StringReader(algTxt));
stok.CommentChar('#');
stok.IsEOLSignificant = false;
stok.QuoteChar('"');
stok.QuoteChar('\'');
stok.OrdinaryChar('/');
stok.OrdinaryChar('(');
stok.OrdinaryChar(')');
bool colonOk = false;
bool isDisableCountNextTask = false; // only for primitive tasks
currSequence.Depth = 0;
while (stok.NextToken() != StreamTokenizer.TT_EOF)
{
switch (stok.TokenType)
{
case StreamTokenizer.TT_WORD:
string s = stok.StringValue;
PerfTask task = (PerfTask)Activator.CreateInstance(TaskClass(config, s), runData);
task.AlgLineNum = stok.LineNumber;
task.DisableCounting = isDisableCountNextTask;
isDisableCountNextTask = false;
currSequence.AddTask(task);
if (task is RepSumByPrefTask)
{
stok.NextToken();
string prefix = stok.StringValue;
if (prefix == null || prefix.Length == 0)
{
throw new Exception("named report prefix problem - " + stok.ToString());
}
((RepSumByPrefTask)task).SetPrefix(prefix);
}
// check for task param: '(' someParam ')'
stok.NextToken();
if (stok.TokenType != '(')
{
stok.PushBack();
}
else
{
// get params, for tasks that supports them - allow recursive parenthetical expressions
stok.IsEOLSignificant = true; // Allow params tokenizer to keep track of line number
StringBuilder @params = new StringBuilder();
stok.NextToken();
if (stok.TokenType != ')')
{
int count = 1;
while (true)
{
switch (stok.TokenType)
{
case StreamTokenizer.TT_NUMBER:
{
@params.Append(stok.NumberValue);
break;
}
case StreamTokenizer.TT_WORD:
{
@params.Append(stok.StringValue);
break;
}
case StreamTokenizer.TT_EOF:
{
throw new Exception("Unexpexted EOF: - " + stok.ToString());
}
case '"':
case '\'':
{
@params.Append((char)stok.TokenType);
// re-escape delimiters, if any
@params.Append(stok.StringValue.Replace("" + (char)stok.TokenType, @"\" + (char)stok.TokenType));
@params.Append((char)stok.TokenType);
break;
}
case '(':
{
@params.Append((char)stok.TokenType);
++count;
break;
}
case ')':
{
if (--count >= 1)
{ // exclude final closing parenthesis
@params.Append((char)stok.TokenType);
}
else
{
goto BALANCED_PARENS_BREAK;
}
break;
}
default:
{
@params.Append((char)stok.TokenType);
break;
}
}
stok.NextToken();
}
BALANCED_PARENS_BREAK : { }
}
stok.IsEOLSignificant = false;
string prm = @params.ToString().Trim();
if (prm.Length > 0)
{
task.SetParams(prm);
}
}
// ---------------------------------------
colonOk = false; prevTask = task;
break;
default:
char c = (char)stok.TokenType;
switch (c)
{
case ':':
if (!colonOk)
{
throw new Exception("colon unexpexted: - " + stok.ToString());
}
colonOk = false;
// get repetitions number
stok.NextToken();
if ((char)stok.TokenType == '*')
{
((TaskSequence)prevTask).SetRepetitions(TaskSequence.REPEAT_EXHAUST);
}
else
{
if (stok.TokenType != StreamTokenizer.TT_NUMBER)
{
throw new Exception("expected repetitions number or XXXs: - " + stok.ToString());
}
else
{
double num = stok.NumberValue;
stok.NextToken();
if (stok.TokenType == StreamTokenizer.TT_WORD && stok.StringValue.Equals("s", StringComparison.Ordinal))
{
((TaskSequence)prevTask).SetRunTime(num);
}
else
{
stok.PushBack();
((TaskSequence)prevTask).SetRepetitions((int)num);
}
}
}
// check for rate specification (ops/min)
stok.NextToken();
if (stok.TokenType != ':')
{
stok.PushBack();
}
else
{
// get rate number
stok.NextToken();
if (stok.TokenType != StreamTokenizer.TT_NUMBER)
{
throw new Exception("expected rate number: - " + stok.ToString());
}
// check for unit - min or sec, sec is default
stok.NextToken();
if (stok.TokenType != '/')
{
stok.PushBack();
((TaskSequence)prevTask).SetRate((int)stok.NumberValue, false); // set rate per sec
}
else
{
stok.NextToken();
if (stok.TokenType != StreamTokenizer.TT_WORD)
{
throw new Exception("expected rate unit: 'min' or 'sec' - " + stok.ToString());
}
string unit = stok.StringValue.ToLowerInvariant();
if ("min".Equals(unit, StringComparison.Ordinal))
{
((TaskSequence)prevTask).SetRate((int)stok.NumberValue, true); // set rate per min
}
else if ("sec".Equals(unit, StringComparison.Ordinal))
{
((TaskSequence)prevTask).SetRate((int)stok.NumberValue, false); // set rate per sec
}
else
{
throw new Exception("expected rate unit: 'min' or 'sec' - " + stok.ToString());
}
}
}
colonOk = false;
break;
case '{':
case '[':
// a sequence
// check for sequence name
string name = null;
stok.NextToken();
if (stok.TokenType != '"')
{
stok.PushBack();
}
else
{
name = stok.StringValue;
if (stok.TokenType != '"' || name == null || name.Length == 0)
{
throw new Exception("sequence name problem - " + stok.ToString());
}
}
// start the sequence
TaskSequence seq2 = new TaskSequence(runData, name, currSequence, c == '[');
currSequence.AddTask(seq2);
currSequence = seq2;
colonOk = false;
break;
case '&':
if (currSequence.IsParallel)
{
throw new Exception("Can only create background tasks within a serial task");
}
stok.NextToken();
int deltaPri;
if (stok.TokenType != StreamTokenizer.TT_NUMBER)
{
stok.PushBack();
deltaPri = 0;
}
else
{
// priority
deltaPri = (int)stok.NumberValue;
}
if (prevTask == null)
{
throw new Exception("& was unexpected");
}
else if (prevTask.RunInBackground)
{
throw new Exception("double & was unexpected");
}
else
{
prevTask.SetRunInBackground(deltaPri);
}
break;
case '>':
currSequence.SetNoChildReport(); /* intentional fallthrough */
// end sequence
colonOk = true; prevTask = currSequence;
currSequence = currSequence.Parent;
break;
case '}':
case ']':
// end sequence
colonOk = true; prevTask = currSequence;
currSequence = currSequence.Parent;
break;
case '-':
isDisableCountNextTask = true;
break;
} //switch(c)
break;
} //switch(stok.ttype)
}
if (sequence != currSequence)
{
throw new Exception("Unmatched sequences");
}
// remove redundant top level enclosing sequences
while (sequence.IsCollapsable && sequence.Repetitions == 1 && sequence.GetRate() == 0)
{
IList <PerfTask> t = sequence.Tasks;
if (t != null && t.Count == 1)
{
PerfTask p = t[0];
if (p is TaskSequence)
{
sequence = (TaskSequence)p;
continue;
}
}
break;
}
}
/// <exception cref="System.IO.IOException"></exception>
public static AList<Lexeme> Tokenize(string s)
{
StreamTokenizer tokenizer = new StreamTokenizer(new StringReader(s));
tokenizer.OrdinaryChar('-');
// Don't parse minus as part of numbers.
AList<Lexeme> tokBuf = new AList<Lexeme>();
while (tokenizer.NextToken() != StreamTokenizer.TT_EOF)
{
switch (tokenizer.ttype)
{
case StreamTokenizer.TT_NUMBER:
{
tokBuf.AddItem(new Lexeme(Lexeme.NUMBER, tokenizer.sval.ToString()));
break;
}
case StreamTokenizer.TT_WORD:
{
tokBuf.AddItem(new Lexeme(Lexeme.WORD, tokenizer.sval));
break;
}
default:
{
// operator
if ((char)tokenizer.ttype.ToString().Equals("("))
{
tokBuf.AddItem(new Lexeme(Lexeme.LPAREN, (char)tokenizer.ttype.ToString()));
}
else
{
if ((char)tokenizer.ttype.ToString().Equals(")"))
{
tokBuf.AddItem(new Lexeme(Lexeme.RPAREN, (char)tokenizer.ttype.ToString()));
}
else
{
if ((char)tokenizer.ttype.ToString().Equals(","))
{
tokBuf.AddItem(new Lexeme(Lexeme.COMMA, (char)tokenizer.ttype.ToString()));
}
else
{
tokBuf.AddItem(new Lexeme(Lexeme.OPERATOR, (char)tokenizer.ttype.ToString()));
}
}
}
break;
break;
}
}
}
return tokBuf;
}
/// <summary>
/// Reads a set of coordinates into ChemFrame.
/// </summary>
/// <param name="model"></param>
private void ReadCoordinates(IChemModel model)
{
var moleculeSet = model.Builder.NewAtomContainerSet();
IAtomContainer molecule = model.Builder.NewAtomContainer();
string line = input.ReadLine();
line = input.ReadLine();
line = input.ReadLine();
line = input.ReadLine();
while (true)
{
line = input.ReadLine();
if ((line == null) || (line.Contains("-----")))
{
break;
}
int atomicNumber;
StringReader sr = new StringReader(line);
StreamTokenizer token = new StreamTokenizer(sr);
token.NextToken();
// ignore first token
if (token.NextToken() == StreamTokenizer.TTypeNumber)
{
atomicNumber = (int)token.NumberValue;
if (atomicNumber == 0)
{
// Skip dummy atoms. Dummy atoms must be skipped
// if frequencies are to be read because Gaussian
// does not report dummy atoms in frequencies, and
// the number of atoms is used for reading frequencies.
continue;
}
}
else
{
throw new CDKException("Error while reading coordinates: expected integer.");
}
token.NextToken();
// ignore third token
double x;
double y;
double z;
if (token.NextToken() == StreamTokenizer.TTypeNumber)
{
x = token.NumberValue;
}
else
{
throw new IOException("Error reading x coordinate");
}
if (token.NextToken() == StreamTokenizer.TTypeNumber)
{
y = token.NumberValue;
}
else
{
throw new IOException("Error reading y coordinate");
}
if (token.NextToken() == StreamTokenizer.TTypeNumber)
{
z = token.NumberValue;
}
else
{
throw new IOException("Error reading z coordinate");
}
string symbol = "Du";
symbol = PeriodicTable.GetSymbol(atomicNumber);
IAtom atom = model.Builder.NewAtom(symbol);
atom.Point3D = new Vector3(x, y, z);
molecule.Atoms.Add(atom);
}
// this is the place where we store the atomcount to be used as a
// counter in the nmr reading
atomCount = molecule.Atoms.Count;
moleculeSet.Add(molecule);
model.MoleculeSet = moleculeSet;
}
/// <summary>
/// Instantiates the given analysis factory class after pulling params from
/// the given stream tokenizer, then stores the result in the appropriate
/// pipeline component list.
/// </summary>
/// <param name="stok">Stream tokenizer from which to draw analysis factory params.</param>
/// <param name="clazz">Analysis factory class to instantiate.</param>
private void CreateAnalysisPipelineComponent(StreamTokenizer stok, Type clazz)
{
IDictionary <string, string> argMap = new Dictionary <string, string>();
bool parenthetical = false;
try
{
while (stok.NextToken() != StreamTokenizer.TokenType_EndOfStream)
{
switch (stok.TokenType)
{
case ',':
{
if (parenthetical)
{
// Do nothing
break;
}
else
{
// Finished reading this analysis factory configuration
goto WHILE_LOOP_BREAK;
}
}
case '(':
{
if (parenthetical)
{
throw RuntimeException.Create
("Line #" + GetLineNumber(stok) + ": Unexpected opening parenthesis.");
}
parenthetical = true;
break;
}
case ')':
{
if (parenthetical)
{
parenthetical = false;
}
else
{
throw RuntimeException.Create
("Line #" + GetLineNumber(stok) + ": Unexpected closing parenthesis.");
}
break;
}
case StreamTokenizer.TokenType_Word:
{
if (!parenthetical)
{
throw RuntimeException.Create("Line #" + GetLineNumber(stok) + ": Unexpected token '" + stok.StringValue + "'");
}
string argName = stok.StringValue;
stok.NextToken();
if (stok.TokenType != ':')
{
throw RuntimeException.Create
("Line #" + GetLineNumber(stok) + ": Missing ':' after '" + argName + "' param to " + clazz.Name);
}
stok.NextToken();
string argValue = stok.StringValue;
switch (stok.TokenType)
{
case StreamTokenizer.TokenType_Number:
{
argValue = stok.NumberValue.ToString(CultureInfo.InvariantCulture);
// Drop the ".0" from numbers, for integer arguments
argValue = TRAILING_DOT_ZERO_PATTERN.Replace(argValue, "", 1);
// Intentional fall-through
argMap[argName] = argValue;
break;
}
case '"':
case '\'':
case StreamTokenizer.TokenType_Word:
{
argMap[argName] = argValue;
break;
}
case StreamTokenizer.TokenType_EndOfStream:
{
throw RuntimeException.Create("Unexpected EOF: " + stok.ToString());
}
default:
{
throw RuntimeException.Create
("Line #" + GetLineNumber(stok) + ": Unexpected token: " + stok.ToString());
}
}
break;
}
}
}
WHILE_LOOP_BREAK : { }
if (!argMap.ContainsKey("luceneMatchVersion"))
{
#pragma warning disable 612, 618
argMap["luceneMatchVersion"] = LuceneVersion.LUCENE_CURRENT.ToString();
#pragma warning restore 612, 618
}
AbstractAnalysisFactory instance;
try
{
instance = (AbstractAnalysisFactory)Activator.CreateInstance(clazz, argMap);
}
catch (Exception e) when(e.IsException())
{
throw RuntimeException.Create("Line #" + GetLineNumber(stok) + ": ", e);
}
if (instance is IResourceLoaderAware resourceLoaderAware)
{
DirectoryInfo baseDir = new DirectoryInfo(RunData.Config.Get("work.dir", "work"));
resourceLoaderAware.Inform(new FilesystemResourceLoader(baseDir));
}
if (typeof(CharFilterFactory).IsAssignableFrom(clazz))
{
charFilterFactories.Add((CharFilterFactory)instance);
}
else if (typeof(TokenizerFactory).IsAssignableFrom(clazz))
{
tokenizerFactory = (TokenizerFactory)instance;
}
else if (typeof(TokenFilterFactory).IsAssignableFrom(clazz))
{
tokenFilterFactories.Add((TokenFilterFactory)instance);
}
}
catch (Exception e) when(e.IsRuntimeException())
{
if (e.Message.StartsWith("Line #", StringComparison.Ordinal))
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
else
{
throw RuntimeException.Create("Line #" + GetLineNumber(stok) + ": ", e);
}
}
catch (Exception t) when(t.IsThrowable())
{
throw RuntimeException.Create("Line #" + GetLineNumber(stok) + ": ", t);
}
}
public void TestWaebricInput()
{
//Set up tokenizer and input for tokenizer
Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
//Tokenize stream and do some test with it
int current = Tokenizer.NextToken();
int position = 1;
while(current != StreamTokenizer.EOF)
{
if (current == StreamTokenizer.LAYOUT)
{
current = Tokenizer.NextToken();
continue; //ignore layout
}
switch (position)
{
case 1: //module
Assert.AreEqual("module", Tokenizer.GetTextValue());
break;
case 2: //test
Assert.AreEqual("test", Tokenizer.GetTextValue());
break;
case 3: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 4: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 5: // /
Assert.AreEqual('/', Tokenizer.GetCharacterValue());
break;
case 6: //index
Assert.AreEqual("index", Tokenizer.GetTextValue());
break;
case 7: //.
Assert.AreEqual('.', Tokenizer.GetCharacterValue());
break;
case 8: //html
Assert.AreEqual("html", Tokenizer.GetTextValue());
break;
case 9: //:
Assert.AreEqual(':', Tokenizer.GetCharacterValue());
break;
case 10: //home
Assert.AreEqual("home", Tokenizer.GetTextValue());
break;
case 11: //(
Assert.AreEqual('(', Tokenizer.GetCharacterValue());
break;
case 12: //)
Assert.AreEqual(')', Tokenizer.GetCharacterValue());
break;
case 13: //end
Assert.AreEqual("end", Tokenizer.GetTextValue());
break;
}
position++;
current = Tokenizer.NextToken();
}
}
public void TestWaebricInput()
{
//Set up tokenizer and input for tokenizer
Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
//Tokenize stream and do some test with it
int current = Tokenizer.NextToken();
int position = 1;
while (current != StreamTokenizer.EOF)
{
if (current == StreamTokenizer.LAYOUT)
{
current = Tokenizer.NextToken();
continue; //ignore layout
}
switch (position)
{
case 1: //module
Assert.AreEqual("module", Tokenizer.GetTextValue());
break;
case 2: //test
Assert.AreEqual("test", Tokenizer.GetTextValue());
break;
case 3: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 4: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 5: // /
Assert.AreEqual('/', Tokenizer.GetCharacterValue());
break;
case 6: //index
Assert.AreEqual("index", Tokenizer.GetTextValue());
break;
case 7: //.
Assert.AreEqual('.', Tokenizer.GetCharacterValue());
break;
case 8: //html
Assert.AreEqual("html", Tokenizer.GetTextValue());
break;
case 9: //:
Assert.AreEqual(':', Tokenizer.GetCharacterValue());
break;
case 10: //home
Assert.AreEqual("home", Tokenizer.GetTextValue());
break;
case 11: //(
Assert.AreEqual('(', Tokenizer.GetCharacterValue());
break;
case 12: //)
Assert.AreEqual(')', Tokenizer.GetCharacterValue());
break;
case 13: //end
Assert.AreEqual("end", Tokenizer.GetTextValue());
break;
}
position++;
current = Tokenizer.NextToken();
}
}
