/// <summary> /// Creates a token stream that tokenizes the given string into token terms /// (aka words). /// </summary> /// <param name="fieldName"> /// the name of the field to tokenize (currently ignored). </param> /// <param name="reader"> /// reader (e.g. charfilter) of the original text. can be null. </param> /// <param name="text"> /// the string to tokenize </param> /// <returns> a new token stream </returns> public TokenStreamComponents CreateComponents(string fieldName, TextReader reader, string text) { // Ideally the Analyzer superclass should have a method with the same signature, // with a default impl that simply delegates to the StringReader flavour. if (reader == null) { reader = new FastStringReader(text); } if (pattern == NON_WORD_PATTERN) // fast path { return new TokenStreamComponents(new FastStringTokenizer(reader, true, toLowerCase, stopWords)); } // fast path else if (pattern == WHITESPACE_PATTERN) { return new TokenStreamComponents(new FastStringTokenizer(reader, false, toLowerCase, stopWords)); } Tokenizer tokenizer = new PatternTokenizer(reader, pattern, toLowerCase); TokenStream result = (stopWords != null) ? (TokenStream)new StopFilter(matchVersion, tokenizer, stopWords) : tokenizer; return new TokenStreamComponents(tokenizer, result); }
/// <summary> /// Creates a token stream that tokenizes the given string into token terms /// (aka words). /// </summary> /// <param name="fieldName"> /// the name of the field to tokenize (currently ignored). </param> /// <param name="reader"> /// reader (e.g. charfilter) of the original text. can be null. </param> /// <param name="text"> /// the string to tokenize </param> /// <returns> a new token stream </returns> public TokenStreamComponents createComponents(string fieldName, TextReader reader, string text) { // Ideally the Analyzer superclass should have a method with the same signature, // with a default impl that simply delegates to the StringReader flavour. if (reader == null) { reader = new FastStringReader(text); } if (pattern == NON_WORD_PATTERN) // fast path { return new TokenStreamComponents(new FastStringTokenizer(reader, true, toLowerCase, stopWords)); } // fast path else if (pattern == WHITESPACE_PATTERN) { return new TokenStreamComponents(new FastStringTokenizer(reader, false, toLowerCase, stopWords)); } Tokenizer tokenizer = new PatternTokenizer(reader, pattern, toLowerCase); TokenStream result = (stopWords != null) ? new StopFilter(matchVersion, tokenizer, stopWords) : tokenizer; return new TokenStreamComponents(tokenizer, result); }
public void Load() { if (_loaded) { return; } _levels = new List <LevelData>(); if (data == null) { data = Resources.Load("leveldata", typeof(TextAsset)) as TextAsset; if (data == null) { D.log("[Levels] level data is null!"); return; } } using (FastStringReader reader = new FastStringReader(data.text)) { bool skip = false; while (!skip) { LevelData one = new LevelData(); one.data = new char[G.rows, G.cols]; for (int i = 0; i < G.rows; i++) { for (int j = 0; j < G.cols; j++) { one.data[i, j] = '-'; } } // here's the big impact for (int i = 0; i < G.rowsInData; i++) { string line; line = reader.ReadLine(); if (line == null) { D.log("[Levels] The data format is corrupted!"); skip = true; break; } line = line.Trim().Replace(" ", ""); for (int j = 0; j < line.Length; j++) { one.data[i, j] = line[j]; } } if (!skip) { _levels.Add(one); // consume the blank line. string line = reader.ReadLine(); if (line == null) { skip = true; } } } } D.log("[Levels] {0} levels found.", _levels.Count); _loaded = true; }