protected string ApplyNestedSubstitutions(string substitution, StringToStringTable substitutions) { if (substitution.IndexOfAny(SubstitutionIdentifierPrefixes) == -1) { return(substitution); } int rounds = 1; string currentResult = substitution; Tokenizer tokenizer = new Tokenizer(currentResult); while (rounds < 50) // safety for infinite recursion { tokenizer = ApplySubstitutions(tokenizer, substitutions, false); if (tokenizer.RawText == currentResult) { break; } currentResult = tokenizer.RawText; rounds++; } return(currentResult); }
/* Function: FindSubstitutions * Searches the source for substitution definitions and returns them as a <StringToStringTable>. */ protected StringToStringTable FindSubstitutions(Tokenizer source) { StringToStringTable substitutions = new StringToStringTable(); TokenIterator iterator = source.FirstToken; string identifier, value, declaration; while (iterator.IsInBounds) { if (TryToSkipSubstitutionDefinition(ref iterator, out identifier, out value, out declaration)) { substitutions.Add(identifier, value); } else { GenericSkip(ref iterator); } } return(substitutions); }
override public string Process(string javascript, bool shrink = true) { Tokenizer source = new Tokenizer(javascript); StringToStringTable substitutions = FindSubstitutions(source); source = ApplySubstitutions(source, substitutions); if (!shrink) { return(source.RawText); } // Search comments for sections to include in the output StringBuilder output = new StringBuilder(javascript.Length); string includeInOutput = FindIncludeInOutput(GetPossibleDocumentationComments(source)); if (includeInOutput != null) { output.AppendLine("/*"); output.Append(includeInOutput); output.AppendLine("*/"); output.AppendLine(); } // Shrink the source TokenIterator iterator = source.FirstToken; string spaceSeparatedSymbols = "+-"; while (iterator.IsInBounds) { char lastChar = (output.Length > 0 ? output[output.Length - 1] : '\0'); if (TryToSkipWhitespace(ref iterator) == true) // includes comments { char nextChar = iterator.Character; if ((spaceSeparatedSymbols.IndexOf(lastChar) != -1 && spaceSeparatedSymbols.IndexOf(nextChar) != -1) || (Tokenizer.FundamentalTypeOf(lastChar) == FundamentalType.Text && Tokenizer.FundamentalTypeOf(nextChar) == FundamentalType.Text)) { output.Append(' '); } } else { TokenIterator prevIterator = iterator; if (TryToSkipString(ref iterator) || TryToSkipRegex(ref iterator)) { source.AppendTextBetweenTo(prevIterator, iterator, output); } else { iterator.AppendTokenTo(output); iterator.Next(); } } } return(output.ToString()); }
/* Function: ApplySubstitutions * Finds all substitutions in the source that match those in the table and replaces them with their values. Will also comment * out any substitution definitions found. */ protected Tokenizer ApplySubstitutions(Tokenizer source, StringToStringTable substitutions, bool applyNestedSubstitutions = true) { TokenIterator iterator = source.FirstToken; // Find the first valid substitution identifier or definition. If there aren't any we don't want to \do unnecessary memory // allocation and processing. bool foundSubstitution = false; string identifier = null; string localeIdentifier, value, declaration; while (iterator.IsInBounds) { if (TryToSkipSubstitutionIdentifier(ref iterator, out identifier) || TryToSkipLocaleSubstitutionIdentifier(ref iterator, out identifier, out localeIdentifier)) { foundSubstitution = true; break; } // else if (TryToSkipSubstitutionDefinition()) // { // Unnecessary because definitions will start with identifiers so it will get picked up by that // } else { GenericSkip(ref iterator); } } if (!foundSubstitution) { return(source); } // Now that we know we have one, we can back up the iterator and build new output iterator.PreviousByCharacters(identifier.Length); StringBuilder output = new StringBuilder(source.RawText.Length); output.Append(source.RawText, 0, iterator.RawTextIndex); while (iterator.IsInBounds) { TokenIterator previousIterator = iterator; if (TryToSkipSubstitutionDefinition(ref iterator, out identifier, out value, out declaration)) { if (this.blockCommentStringPairs != null) { output.Append(this.blockCommentStringPairs[0] + ' ' + declaration + ' ' + this.blockCommentStringPairs[1]); } } else if (TryToSkipSubstitutionIdentifier(ref iterator, out identifier)) { string substitution = substitutions[identifier]; if (substitution == null) { output.Append(identifier); } else { if (applyNestedSubstitutions) { substitution = ApplyNestedSubstitutions(substitution, substitutions); } output.Append(substitution); } } else if (TryToSkipLocaleSubstitutionIdentifier(ref iterator, out identifier, out localeIdentifier)) { string substitution = Engine.Locale.SafeGet("NaturalDocs.Engine", localeIdentifier, null); if (substitution == null) { output.Append(identifier); } else { output.Append('"' + substitution.StringEscape() + '"'); } } else { GenericSkip(ref iterator); source.AppendTextBetweenTo(previousIterator, iterator, output); } } return(new Tokenizer(output.ToString())); }
// Group: Functions // __________________________________________________________________________ /* Constructor: CommandLine */ public CommandLine(string[] commandLineSegments) { segments = commandLineSegments; index = 0; aliases = new StringToStringTable(); }
override public string Process(string css, bool shrink = true) { Tokenizer source = new Tokenizer(css); StringToStringTable substitutions = FindSubstitutions(source); source = ApplySubstitutions(source, substitutions); if (!shrink) { return(source.RawText); } // Search comments for sections to include in the output StringBuilder output = new StringBuilder(css.Length); string includeInOutput = FindIncludeInOutput(GetPossibleDocumentationComments(source)); if (includeInOutput != null) { output.AppendLine("/*"); output.Append(includeInOutput); output.AppendLine("*/"); output.AppendLine(); } // Shrink the source TokenIterator iterator = source.FirstToken; // We have to be more cautious than the JavaScript shrinker. You don't want something like "head .class" to become // "head.class". Colon is a special case because we only want to remove spaces after it ("font-size: 12pt") and not // before ("body :link"). string safeToCondenseAround = "{},;:+>[]= \0\n\r"; while (iterator.IsInBounds) { char lastChar = (output.Length > 0 ? output[output.Length - 1] : '\0'); if (TryToSkipWhitespace(ref iterator)) // includes comments { char nextChar = iterator.Character; if (nextChar == ':' || (safeToCondenseAround.IndexOf(lastChar) == -1 && safeToCondenseAround.IndexOf(nextChar) == -1)) { output.Append(' '); } } else { TokenIterator prevIterator = iterator; if (TryToSkipString(ref iterator)) { source.AppendTextBetweenTo(prevIterator, iterator, output); } else { if (iterator.Character == '}' && lastChar == ';') { // Semicolons are unnecessary at the end of blocks. However, we have to do this here instead of in a // global search and replace for ";}" because we don't want to alter that sequence if it appears in a string. output[output.Length - 1] = '}'; } else { iterator.AppendTokenTo(output); } iterator.Next(); } } } return(output.ToString()); }
/* Function: Load * Loads the translation file at the specified path and returns its data. */ private static StringToStringTable Load(Path fileName) { using (var fileReader = new System.IO.StreamReader(fileName, System.Text.Encoding.UTF8, true)) { StringToStringTable fileContent = new StringToStringTable(KeySettingsForIdentifiers); string line, originalLine; string identifier = ""; // Not necessary to initialize, just need to shut the compiler up. string value = ""; bool inMultiline = false; int lineNumber = 0; int index = -1; while ((originalLine = fileReader.ReadLine()) != null) { line = originalLine.Trim(); lineNumber++; if (inMultiline == true) { if (line == "}}}") { inMultiline = false; fileContent.Add(identifier, value); } else if (line.IndexOf("}}}") != -1) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": Nothing can be on the same line as }}}."); } else { // We want to preserve any leading whitespace. value += originalLine.TrimEnd() + System.Environment.NewLine; } } else // not in multiline { if (line == "" || line[0] == '#') { /* skip */ } else if ((index = line.IndexOf("{{{")) != -1) { identifier = line.Substring(0, index).TrimEnd(); if (String.IsNullOrEmpty(identifier)) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": No identifier before {{{."); } if (identifier.IndexOf(':') != -1) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": Line cannot have a colon and {{{."); } if (line.EndsWith("{{{") == false) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": There cannot be content on the same line as {{{."); } value = ""; inMultiline = true; } else if ((index = line.IndexOf(':')) != -1) { identifier = line.Substring(0, index).TrimEnd(); if (index + 1 < line.Length) { value = line.Substring(index + 1).TrimStart(); } else { value = null; } if (String.IsNullOrEmpty(identifier)) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": No identifier before colon."); } if (String.IsNullOrEmpty(value)) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": No value after colon."); } fileContent.Add(identifier, value); } else { throw new Exceptions.UserFriendly("Error in translation file " + fileName + " line " + lineNumber + ": Unrecognized line format."); } } } // while ReadLine if (inMultiline == true) { throw new Exceptions.UserFriendly("Error in translation file " + fileName + ": Unclosed multiline block."); } return(fileContent); } }
// Group: Private Functions // __________________________________________________________________________ /* Function: Lookup * * Gets the specified string for the passed locale. Will <Load()> language files as needed. * * Parameters: * * module - The module string to load. * identifier - The identifier string to load. * localeCode - The locale string to load. */ private static string Lookup(string module, string identifier, string localeCode) { string moduleLocaleString = module + '.' + localeCode; accessLock.AcquireReaderLock(-1); try { if (!translations.ContainsKey(moduleLocaleString)) { // Load will automatically upgrade it to a writer lock. Load(module, localeCode); } StringToStringTable translation = translations[moduleLocaleString]; if (translation.ContainsKey(identifier)) { // The "finally" section will still run. return(translation[identifier]); } // Check to see if we can strip the specific locale. en-us -> en // We can't use recursion for these without losing the original locale, which we need for the exception. int index = localeCode.IndexOf('-'); if (index != -1) { string parentLocale = localeCode.Substring(0, index); string parentModuleLocaleString = module + '.' + parentLocale; if (!translations.ContainsKey(parentModuleLocaleString)) { Load(module, parentLocale); } translation = translations[parentModuleLocaleString]; if (translation.ContainsKey(identifier)) { return(translation[identifier]); } } // Otherwise check to see if we can go to the default. en -> default if (localeCode != "default") { string defaultModuleLocaleString = module + ".default"; if (!translations.ContainsKey(defaultModuleLocaleString)) { Load(module, "default"); } translation = translations[defaultModuleLocaleString]; if (translation.ContainsKey(identifier)) { return(translation[identifier]); } } // Okay, it doesn't exist. throw new Exception("Identifier \"" + module + '.' + identifier + "\" does not exist in any translation files for the locale " + localeCode + '.'); } finally { accessLock.ReleaseReaderLock(); } }