/********* ** Public methods *********/ /// <summary>Break a raw string into its constituent lexical character patterns.</summary> /// <param name="rawText">The raw text to tokenize.</param> public IEnumerable <LexBit> TokenizeString(string rawText) { // special cases if (rawText == null) { yield break; } if (string.IsNullOrWhiteSpace(rawText)) { yield return(new LexBit(LexBitType.Literal, rawText)); yield break; } // parse string[] parts = this.LexicalSplitPattern.Split(rawText); foreach (string part in parts) { if (part == "") { continue; // split artifact } LexBitType type = part switch { "{{" => LexBitType.StartToken, "}}" => LexBitType.EndToken, InternalConstants.PositionalInputArgSeparator => LexBitType.PositionalInputArgSeparator, InternalConstants.NamedInputArgSeparator => LexBitType.NamedInputArgSeparator, _ => LexBitType.Literal }; yield return(new LexBit(type, part)); } }
/********* ** Public methods *********/ /// <summary>Construct an instance.</summary> /// <param name="type">The lexical character pattern type.</param> /// <param name="text">The raw matched text.</param> public LexBit(LexBitType type, string text) { this.Type = type; this.Text = text; }