/* Function: AddModifierBlock * Adds a full modifier block marked with <PrototypeParsingType.OpeningTypeModifier> or <PrototypeParsingType.OpeningParamModifier> * to the type builder. */ public void AddModifierBlock(TokenIterator openingToken, TokenIterator closingToken) { if ((openingToken.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier && openingToken.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier) || (closingToken.PrototypeParsingType != PrototypeParsingType.ClosingTypeModifier && closingToken.PrototypeParsingType != PrototypeParsingType.ClosingParamModifier)) { throw new InvalidOperationException(); } TokenIterator iterator = openingToken; TokenIterator end = closingToken; end.Next(); while (iterator < end) { FundamentalType thisTokenType = (iterator.Character == '_' ? FundamentalType.Text : iterator.FundamentalType); // Do we need to add a space? if ((thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Text && (iterator.Tokenizer != lastTokenIterator.Tokenizer || iterator.TokenIndex != lastTokenIterator.TokenIndex + 1)) || (thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Symbol && !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock))) { rawText.Append(' '); prototypeParsingTypes.Add(PrototypeParsingType.Null); syntaxHighlightingTypes.Add(SyntaxHighlightingType.Null); } while (iterator < end) { iterator.AppendTokenTo(rawText); prototypeParsingTypes.Add(iterator.PrototypeParsingType); syntaxHighlightingTypes.Add(iterator.SyntaxHighlightingType); iterator.Next(); } lastTokenIterator = closingToken; lastTokenType = thisTokenType; lastSymbolWasBlock = true; dontAddSpaceAfterSymbol = false; if (thisTokenType == FundamentalType.Text) { pastFirstText = true; } } }
override public string Process(string javascript, bool shrink = true) { Tokenizer source = new Tokenizer(javascript); StringToStringTable substitutions = FindSubstitutions(source); source = ApplySubstitutions(source, substitutions); if (!shrink) { return(source.RawText); } // Search comments for sections to include in the output StringBuilder output = new StringBuilder(javascript.Length); string includeInOutput = FindIncludeInOutput(GetPossibleDocumentationComments(source)); if (includeInOutput != null) { output.AppendLine("/*"); output.Append(includeInOutput); output.AppendLine("*/"); output.AppendLine(); } // Shrink the source TokenIterator iterator = source.FirstToken; string spaceSeparatedSymbols = "+-"; while (iterator.IsInBounds) { char lastChar = (output.Length > 0 ? output[output.Length - 1] : '\0'); if (TryToSkipWhitespace(ref iterator) == true) // includes comments { char nextChar = iterator.Character; if ((spaceSeparatedSymbols.IndexOf(lastChar) != -1 && spaceSeparatedSymbols.IndexOf(nextChar) != -1) || (Tokenizer.FundamentalTypeOf(lastChar) == FundamentalType.Text && Tokenizer.FundamentalTypeOf(nextChar) == FundamentalType.Text)) { output.Append(' '); } } else { TokenIterator prevIterator = iterator; if (TryToSkipString(ref iterator) || TryToSkipRegex(ref iterator)) { source.AppendTextBetweenTo(prevIterator, iterator, output); } else { iterator.AppendTokenTo(output); iterator.Next(); } } } return(output.ToString()); }
/* Function: GetJavadocLinkSymbol * Returns the symbol part of a @see or {@link} tag and moves the iterator past it. */ protected string GetJavadocLinkSymbol(ref TokenIterator iterator) { StringBuilder symbol = new StringBuilder(); // In Javadoc, spaces are only allowed in parentheses. We're going to go further and allow them in any braces to support // templates and other languages. However, for angle brackets they must be preceded by a comma. This allows // "Template<A, B>" to be supported while not getting tripped on "operator<". // Most symbols won't have braces so create this on demand. SafeStack <char> braceStack = null; while (iterator.IsInBounds) { if (iterator.Character == '(' || iterator.Character == '[' || iterator.Character == '{' || iterator.Character == '<') { if (braceStack == null) { braceStack = new SafeStack <char>(); } braceStack.Push(iterator.Character); symbol.Append(iterator.Character); } else if ((iterator.Character == ')' && braceStack != null && braceStack.Peek() == '(') || (iterator.Character == ']' && braceStack != null && braceStack.Peek() == '[') || (iterator.Character == '}' && braceStack != null && braceStack.Peek() == '{') || (iterator.Character == '>' && braceStack != null && braceStack.Peek() == '<')) { braceStack.Pop(); symbol.Append(iterator.Character); } else if (iterator.Character == '}' && (braceStack == null || braceStack.Contains('{') == false)) { // If we're at an unopened closing brace we're probably at the end of a {@link}. We check if the stack contains an // opening brace instead of checking whether it's empty to ignore any possible opening angle brackets that could // screw us up like "operator<". break; } else if (iterator.FundamentalType == FundamentalType.Text || iterator.FundamentalType == FundamentalType.Symbol) { iterator.AppendTokenTo(symbol); } else if (iterator.FundamentalType == FundamentalType.Whitespace) { if (braceStack == null || braceStack.Count == 0) { break; } else if (braceStack.Peek() == '<') { TokenIterator lookbehind = iterator; lookbehind.Previous(); if (lookbehind.Character == ',') { iterator.AppendTokenTo(symbol); } else { break; } } else { iterator.AppendTokenTo(symbol); } } else // line break { break; } iterator.Next(); } // Javadoc uses Class.Class#Member. First remove leading hashes to handle just #Member while (symbol.Length > 0 && symbol[0] == '#') { symbol.Remove(0, 1); } // Convert any remaining hashes to dots. Ideally we would use the language's native member operator but it's not // easy to get it here. return(symbol.ToString().Replace('#', '.')); }
override public string Process(string css, bool shrink = true) { Tokenizer source = new Tokenizer(css); StringToStringTable substitutions = FindSubstitutions(source); source = ApplySubstitutions(source, substitutions); if (!shrink) { return(source.RawText); } // Search comments for sections to include in the output StringBuilder output = new StringBuilder(css.Length); string includeInOutput = FindIncludeInOutput(GetPossibleDocumentationComments(source)); if (includeInOutput != null) { output.AppendLine("/*"); output.Append(includeInOutput); output.AppendLine("*/"); output.AppendLine(); } // Shrink the source TokenIterator iterator = source.FirstToken; // We have to be more cautious than the JavaScript shrinker. You don't want something like "head .class" to become // "head.class". Colon is a special case because we only want to remove spaces after it ("font-size: 12pt") and not // before ("body :link"). string safeToCondenseAround = "{},;:+>[]= \0\n\r"; while (iterator.IsInBounds) { char lastChar = (output.Length > 0 ? output[output.Length - 1] : '\0'); if (TryToSkipWhitespace(ref iterator)) // includes comments { char nextChar = iterator.Character; if (nextChar == ':' || (safeToCondenseAround.IndexOf(lastChar) == -1 && safeToCondenseAround.IndexOf(nextChar) == -1)) { output.Append(' '); } } else { TokenIterator prevIterator = iterator; if (TryToSkipString(ref iterator)) { source.AppendTextBetweenTo(prevIterator, iterator, output); } else { if (iterator.Character == '}' && lastChar == ';') { // Semicolons are unnecessary at the end of blocks. However, we have to do this here instead of in a // global search and replace for ";}" because we don't want to alter that sequence if it appears in a string. output[output.Length - 1] = '}'; } else { iterator.AppendTokenTo(output); } iterator.Next(); } } } return(output.ToString()); }
/* Function: AddToken * Adds a single token to the type builder. You must use <AddBlock()> for blocks. */ public void AddToken(TokenIterator iterator) { if (iterator.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier || iterator.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier) { throw new InvalidOperationException(); } if (iterator.PrototypeParsingType == PrototypeParsingType.Type || iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier || iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier || iterator.PrototypeParsingType == PrototypeParsingType.ParamModifier || iterator.PrototypeParsingType == PrototypeParsingType.StartOfTuple || iterator.PrototypeParsingType == PrototypeParsingType.EndOfTuple || iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberSeparator || iterator.PrototypeParsingType == PrototypeParsingType.TupleMemberName) { FundamentalType thisTokenType = (iterator.Character == '_' ? FundamentalType.Text : iterator.FundamentalType); // ignore whitespace, we'll create it as necessary if (thisTokenType == FundamentalType.Text || thisTokenType == FundamentalType.Symbol) { // Do we need to add a space? if ((thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Text && (iterator.Tokenizer != lastTokenIterator.Tokenizer || iterator.TokenIndex != lastTokenIterator.TokenIndex + 1)) || (thisTokenType == FundamentalType.Text && lastTokenType == FundamentalType.Symbol && !dontAddSpaceAfterSymbol && (pastFirstText || lastSymbolWasBlock)) || (!lastTokenIterator.IsNull && lastTokenIterator.PrototypeParsingType == PrototypeParsingType.TupleMemberSeparator)) { rawText.Append(' '); prototypeParsingTypes.Add(PrototypeParsingType.Null); syntaxHighlightingTypes.Add(SyntaxHighlightingType.Null); } // Special handling for package separators and a few other things if (iterator.FundamentalType == FundamentalType.Symbol) { if (iterator.Character == '.' || iterator.MatchesAcrossTokens("::") || (iterator.Character == ':' && dontAddSpaceAfterSymbol) || // second colon of :: iterator.Character == '%' || // used for MyVar%TYPE or MyTable%ROWTYPE in Oracle's PL/SQL iterator.Character == '"' || iterator.Character == '\'' || // strings in Java annotations like @copyright("me") iterator.Character == '@' || // tags in Java annotations like @copyright iterator.Character == '(') // parens around tuples { dontAddSpaceAfterSymbol = true; } else { dontAddSpaceAfterSymbol = false; } } iterator.AppendTokenTo(rawText); prototypeParsingTypes.Add(iterator.PrototypeParsingType); syntaxHighlightingTypes.Add(iterator.SyntaxHighlightingType); lastTokenIterator = iterator; lastTokenType = thisTokenType; lastSymbolWasBlock = false; if (thisTokenType == FundamentalType.Text) { pastFirstText = true; } } } }