示例#1
0
	/* XXX Called from Preprocessor [ugly]. */
	internal static void escape(StringBuilder buf, string cs) {
	    if (cs == null)
	    {
	        return;
	    }
		for (int i = 0; i < cs.length(); i++) {
			char	c = cs.charAt(i);
			switch (c) {
				case '\\':
					buf.append("\\\\");
					break;
				case '"':
					buf.append("\\\"");
					break;
				case '\n':
					buf.append("\\n");
					break;
				case '\r':
					buf.append("\\r");
					break;
				default:
					buf.append(c);
                    break;
			}
		}
	}
 public void DoTest(StringBuilder failQueries)
 {
     bool pass = false;
     SrndQuery lq = null;
     try
     {
         lq = Parser.QueryParser.Parse(queryText);
         if (verbose)
         {
             Console.WriteLine("Query: " + queryText + "\nParsed as: " + lq.ToString());
         }
     }
     catch (ParseException e)
     {
         if (verbose)
         {
             Console.WriteLine("Parse exception for query:\n"
                               + queryText + "\n"
                               + e.Message);
         }
         pass = true;
     }
     if (!pass)
     {
         failQueries.append(queryText);
         failQueries.append("\nParsed as: ");
         failQueries.append(lq.toString());
         failQueries.append("\n");
     }
 }
 /**
  * Returns the message string of the DuplicateFormatFlagsException.
  *
  * @return the message string of the DuplicateFormatFlagsException.
  */
 public override String getMessage()
 {
     StringBuilder buffer = new StringBuilder();
     buffer.append("Flags of the DuplicateFormatFlagsException is'");
     buffer.append(flags);
     buffer.append("'");
     return buffer.toString();
 }
示例#4
0
        public override String ToString()
        {
            StringBuilder sb = new StringBuilder();

            sb.append("[");
            sb.append(getAgentLocation());
            sb.append(", ");
            sb.append(getLocationState());
            sb.append("]");

            return sb.ToString();
        }
示例#5
0
	private void concat(StringBuilder buf, Argument arg) {
		Iterator<Token>	it = arg.iterator();
		while (it.hasNext()) {
			Token	tok = it.next();
			buf.append(tok.getText());
		}
	}
 public static String toASCIILowerCase(String s)
 {
     int len = s.length();
     StringBuilder buffer = new StringBuilder(len);
     for (int i = 0; i < len; i++)
     {
         char c = s.charAt(i);
         if ('A' <= c && c <= 'Z')
         {
             buffer.append((char)(c + ('a' - 'A')));
         }
         else
         {
             buffer.append(c);
         }
     }
     return buffer.toString();
 }
示例#7
0
        internal const String encoding = "utf-8"; //$NON-NLS-1$

        #endregion Fields

        #region Methods

        /**
         * All characters except letters ('a'..'z', 'A'..'Z') and numbers ('0'..'9')
         * and legal characters are converted into their hexidecimal value prepended
         * by '%'.
         * <p>
         * For example: '#' -> %23
         * Other characters, which are unicode chars that are not US-ASCII, and are
         * not ISO Control or are not ISO Space chars, are preserved.
         * <p>
         * Called from {@code URI.quoteComponent()} (for multiple argument
         * constructors)
         *
         * @param s
         *            java.lang.String the string to be converted
         * @param legal
         *            java.lang.String the characters allowed to be preserved in the
         *            string s
         * @return java.lang.String the converted string
         */
        internal static String quoteIllegal(String s, String legal)
        {
            //throws UnsupportedEncodingException {
            StringBuilder buf = new StringBuilder();
            for (int i = 0; i < s.length(); i++) {
                char ch = s.charAt(i);
                if ((ch >= 'a' && ch <= 'z')
                        || (ch >= 'A' && ch <= 'Z')
                        || (ch >= '0' && ch <= '9')
                        || legal.indexOf(ch) > -1
                        || (ch > 127 && !java.lang.Character.isSpaceChar(ch) && !java.lang.Character
                                .isISOControl(ch))) {
                    buf.append(ch);
                } else {
                    byte[] bytes = new String(new char[] { ch }).getBytes(encoding);
                    for (int j = 0; j < bytes.Length; j++) {
                        buf.append('%');
                        buf.append(digits.charAt((bytes[j] & 0xf0) >> 4));
                        buf.append(digits.charAt(bytes[j] & 0xf));
                    }
                }
            }
            return buf.toString();
        }
 private KeyValuePair<List<List<string>>, string> generateFileInput(int count, string fieldDelimiter, bool hasWeights, bool hasPayloads)
 {
     List<List<string>> entries = new List<List<string>>();
     StringBuilder sb = new StringBuilder();
     bool hasPayload = hasPayloads;
     for (int i = 0; i < count; i++)
     {
         if (hasPayloads)
         {
             hasPayload = (i == 0) ? true : Random().nextBoolean();
         }
         KeyValuePair<List<string>, string> entrySet = GenerateFileEntry(fieldDelimiter, (!hasPayloads && hasWeights) ? Random().nextBoolean() : hasWeights, hasPayload);
         entries.Add(entrySet.Key);
         sb.append(entrySet.Value);
     }
     return new KeyValuePair<List<List<string>>, string>(entries, sb.toString());
 }
 private KeyValuePair<List<string>, string> GenerateFileEntry(string fieldDelimiter, bool hasWeight, bool hasPayload)
 {
     List<string> entryValues = new List<string>();
     StringBuilder sb = new StringBuilder();
     string term = TestUtil.RandomSimpleString(Random(), 1, 300);
     sb.append(term);
     entryValues.Add(term);
     if (hasWeight)
     {
         sb.append(fieldDelimiter);
         long weight = TestUtil.NextLong(Random(), long.MinValue, long.MaxValue);
         sb.append(weight);
         entryValues.Add(weight.ToString());
     }
     if (hasPayload)
     {
         sb.append(fieldDelimiter);
         string payload = TestUtil.RandomSimpleString(Random(), 1, 300);
         sb.append(payload);
         entryValues.Add(payload);
     }
     sb.append("\n");
     return new KeyValuePair<List<string>, string>(entryValues, sb.toString());
 }
        private static void WriteAndReadAString()
        {
            // Write out a string whose UTF-8 encoding is quite possibly
            // longer than 65535 bytes
            int length = Random().nextInt(A_NUMBER_NEAR_65535) + 1;
            MemoryStream baos = new MemoryStream();
            StringBuilder testBuffer = new StringBuilder();
            for (int i = 0; i < length; i++)
                testBuffer.append((char)Random().Next());
            string testString = testBuffer.toString();
            DataOutputStream dos = new DataOutputStream(baos);
            dos.WriteUTF(testString);

            // Corrupt the data to produce malformed characters
            byte[] testBytes = baos.ToArray();
            int dataLength = testBytes.Length;
            int corruptions = Random().nextInt(MAX_CORRUPTIONS_PER_CYCLE);
            for (int i = 0; i < corruptions; i++)
            {
                int index = Random().nextInt(dataLength);
                testBytes[index] = (byte)Random().Next();
            }

            // Pay special attention to mangling the end to produce
            // partial characters at end
            testBytes[dataLength - 1] = (byte)Random().Next();
            testBytes[dataLength - 2] = (byte)Random().Next();

            // Attempt to decode the bytes back into a String
            MemoryStream bais = new MemoryStream(testBytes);
            DataInputStream dis = new DataInputStream(bais);
            dis.ReadUTF();
        }
示例#11
0
 /// <summary>
 /// Write a hex string of the contents of buffer from position to limit to the
 /// output.
 /// </summary>
 ///
 /// <param name="buffer">The buffer.</param>
 /// <returns>A string of hex bytes.</returns>
 /// <param name="output">The StringBuffer to write to.</param>
 public static void toHex(ByteBuffer buffer, StringBuilder output)
 {
     for (int i = buffer.position(); i < buffer.limit(); ++i) {
         String hex = ILOG.J2CsMapping.Util.IlNumber.ToString((int) buffer.get(i) & 0xff,16);
         if (hex.Length <= 1)
             // Append the leading zero.
             output.append("0");
         output.append(hex);
     }
 }
示例#12
0
        private int buildEclipse()
        {
            System.out.println();
            System.out.println("Eclipse Plugin");
            System.out.println("--------------------------");

            var args = new ArrayList <string>();

            args.add("-al:bin/stabal.jar");
            args.add("-resources:eclipse/resources");

            var    tmpManifest = new File("eclipse/MANIFEST.MF.TMP");
            string version     = "1.0.0";

            using (var reader = new BufferedReader(new FileReader("eclipse/MANIFEST.MF"))) {
                using (var writer = new BufferedWriter(new FileWriter(tmpManifest))) {
                    string line;
                    while ((line = reader.readLine()) != null)
                    {
                        if (line.startsWith("Bundle-Version:"))
                        {
                            int idx = line.indexOf(".qualifier");
                            var sb  = new StringBuilder();
                            sb.append(line.substring("Bundle-Version:".length(), idx + 1).trim());
                            writer.write(line.substring(0, idx + 1));
                            var qualifier = string.format("%1$tY%1$tm%1$td%1$tH%1$tM", GregorianCalendar.getInstance());
                            sb.append(qualifier);
                            writer.write(qualifier);
                            version = sb.toString();
                        }
                        else
                        {
                            writer.write(line);
                        }
                        writer.newLine();
                    }
                }
            }

            args.add("-manifest:eclipse/MANIFEST.MF.TMP");
            args.add("-out:eclipse/stab.tools.eclipse_" + version + ".jar");
            args.add("-define:TRACE");

            var pluginDir = new File("eclipse/eclipse/plugins");
            var files     = pluginDir.listFiles();
            var sb        = new StringBuilder();

            sb.append("-cp:bin/stabrt.jar;bin/stabc.jar;bin/asm-3.3.jar");
            foreach (var file in files)
            {
                if (!file.isDirectory() && file.getName().endsWith(".jar"))
                {
                    sb.append(';');
                    sb.append(file.getAbsolutePath());
                }
            }
            args.add(sb.toString());
            addSourceFiles(new File("eclipse/sources"), args);

            int result = Compiler.runWith(args);

            tmpManifest.delete();
            return(result);
        }
        public static String addColonsToMACAddress(String address)
        {
            StringBuilder sb = new StringBuilder(MAC_ADDRESS_LENGTH);

            sb.append(address.Substring(0, 2));
            sb.append(':');
            sb.append(address.Substring(2, 2));
            sb.append(':');
            sb.append(address.Substring(4, 2));
            sb.append(':');
            sb.append(address.Substring(6, 2));
            sb.append(':');
            sb.append(address.Substring(8, 2));
            sb.append(':');
            sb.append(address.Substring(10, 2));

            return(sb.toString());
        }
示例#14
0
        /**
         * Decodes the string argument which is assumed to be encoded in the {@code
         * x-www-form-urlencoded} MIME content type using the UTF-8 encoding scheme.
         * <p>
         *'%' and two following hex digit characters are converted to the
         * equivalent byte value. All other characters are passed through
         * unmodified.
         * <p>
         * e.g. "A%20B%20C %24%25" -> "A B C $%"
         * <p>
         * Called from URI.getXYZ() methods
         *
         * @param s
         *            java.lang.String The encoded string.
         * @return java.lang.String The decoded version.
         */
        static String decode(String s)
        {
            //throws UnsupportedEncodingException {

            StringBuilder result = new StringBuilder();
            java.io.ByteArrayOutputStream outJ = new java.io.ByteArrayOutputStream();
            for (int i = 0; i < s.length();) {
                char c = s.charAt(i);
                if (c == '%') {
                    outJ.reset();
                    do {
                        if (i + 2 >= s.length()) {
                            throw new java.lang.IllegalArgumentException("Incomplete % sequence at: "+ i); //$NON-NLS-1$
                        }
                        int d1 = java.lang.Character.digit(s.charAt(i + 1), 16);
                        int d2 = java.lang.Character.digit(s.charAt(i + 2), 16);
                        if (d1 == -1 || d2 == -1) {
                            throw new java.lang.IllegalArgumentException("Invalid % sequence ("+s.substring(i, i + 3)+") at: "+java.lang.StringJ.valueOf(i));
                        }
                        outJ.write((byte) ((d1 << 4) + d2));
                        i += 3;
                    } while (i < s.length() && s.charAt(i) == '%');
                    result.append(outJ.toString(encoding));
                    continue;
                }
                result.append(c);
                i++;
            }
            return result.toString();
        }
示例#15
0
 /**
  * Returns a string containing a concise, human-readable description of the
  * receiver.
  *
  * @return a comma delimited list of the indices of all bits that are set.
  */
 public override String ToString()
 {
     StringBuilder sb = new StringBuilder(bits.Length / 2);
     int bitCount = 0;
     sb.append('{');
     bool comma = false;
     for (int i = 0; i < bits.Length; i++) {
         if (bits[i] == 0) {
             bitCount += ELM_SIZE;
             continue;
         }
         for (int j = 0; j < ELM_SIZE; j++) {
             if (((bits[i] & (TWO_N_ARRAY[j])) != 0)) {
                 if (comma) {
                     sb.append(", "); //$NON-NLS-1$
                 }
                 sb.append(bitCount);
                 comma = true;
             }
             bitCount++;
         }
     }
     sb.append('}');
     return sb.toString();
 }
示例#16
0
 private void valueToString(ValueExpression value, StringBuilder sb)
 {
     sb.append("#Value#").append(value.Type).append("#").append((value.Value == this) ? "this" : value.Value).append("#");
 }
示例#17
0
        private void unaryToString(UnaryExpression unary, StringBuilder sb)
        {
            switch (unary.Operator)
            {
            case Cast:
                sb.append("(").append(unary.Type).append(")");
                break;

            case Complement:
                sb.append("~");
                break;

            case Minus:
                sb.append("-");
                break;

            case Not:
                sb.append("!");
                break;

            case Plus:
                sb.append("+");
                break;

            case PreDecrement:
                sb.append("--");
                break;

            case PreIncrement:
                sb.append("++");
                break;

            case Sizeof:
                sb.append("sizeof(");
                break;
            }
            expressionToString(unary.Operand, sb, "");
            switch (unary.Operator)
            {
            case As:
                sb.append(" as ").append(unary.Type);
                break;

            case PostDecrement:
                sb.append("--");
                break;

            case PostIncrement:
                sb.append("++");
                break;

            case Sizeof:
                sb.append(")");
                break;
            }
        }
示例#18
0
 private void fieldToString(FieldExpression field, StringBuilder sb)
 {
     expressionToString(field.Target, sb, "");
     sb.append(".").append(field.Field.getName());
 }
示例#19
0
        private void binaryToString(BinaryExpression binary, StringBuilder sb)
        {
            expressionToString(binary.Left, sb, "");
            switch (binary.Operator)
            {
            case Add:
                sb.append(" + ");
                break;

            case AddAssign:
                sb.append(" += ");
                break;

            case And:
                sb.append(" & ");
                break;

            case AndAssign:
                sb.append(" &= ");
                break;

            case Assign:
                sb.append(" = ");
                break;

            case Divide:
                sb.append(" / ");
                break;

            case DivideAssign:
                sb.append(" /= ");
                break;

            case Element:
                sb.append("[");
                break;

            case Equal:
                sb.append("==");
                break;

            case GreaterThan:
                sb.append(" > ");
                break;

            case GreaterThanOrEqual:
                sb.append(" >= ");
                break;

            case LeftShift:
                sb.append(" << ");
                break;

            case LeftShiftAssign:
                sb.append(" <<= ");
                break;

            case LessThan:
                sb.append(" < ");
                break;

            case LessThanOrEqual:
                sb.append(" <= ");
                break;

            case LogicalAnd:
                sb.append(" && ");
                break;

            case LogicalOr:
                sb.append(" || ");
                break;

            case Modulo:
                sb.append(" % ");
                break;

            case ModuloAssign:
                sb.append(" %= ");
                break;

            case Multiply:
                sb.append(" * ");
                break;

            case MultiplyAssign:
                sb.append(" *= ");
                break;

            case NotEqual:
                sb.append(" != ");
                break;

            case NullCoalescing:
                sb.append(" ?? ");
                break;

            case Or:
                sb.append(" | ");
                break;

            case OrAssign:
                sb.append(" |= ");
                break;

            case RightShift:
                sb.append(" >> ");
                break;

            case RightShiftAssign:
                sb.append(" >>= ");
                break;

            case Subtract:
                sb.append(" - ");
                break;

            case SubtractAssign:
                sb.append(" -= ");
                break;

            case UnsignedRightShift:
                sb.append(" >>> ");
                break;

            case UnsignedRightShiftAssign:
                sb.append(" >>>= ");
                break;

            case Xor:
                sb.append(" ^ ");
                break;

            case XorAssign:
                sb.append(" ^= ");
                break;
            }
            expressionToString(binary.Right, sb, "");
            switch (binary.Operator)
            {
            case Element:
                sb.append("]");
                break;
            }
        }
示例#20
0
        private void statementToString(Statement statement, StringBuilder sb, String indent)
        {
            switch (statement.StatementKind)
            {
            case Block:
                sb.append(indent).append("{\n");
                foreach (var s in ((BlockStatement)statement).Statements)
                {
                    statementToString(s, sb, indent + "    ");
                }
                sb.append(indent).append("}");
                break;

            case Break:
                sb.append(indent).append("break;");
                break;

            case Continue:
                sb.append(indent).append("continue;");
                break;

            case Do:
                var doStatement = (LoopStatement)statement;
                sb.append(indent).append("do\n");
                statementToString(doStatement.Body, sb, indent + "    ");
                sb.append(indent).append("while (");
                expressionToString(doStatement.Condition, sb, "");
                sb.append(");");
                break;

            case Empty:
                sb.append(";");
                break;

            case Goto:
                sb.append(indent).append("goto ");
                labelToString(((GotoStatement)statement).Label, sb);
                sb.append(";");
                break;

            case GotoCase: {
                var gotoCase = (GotoCaseStatement)statement;
                sb.append(indent).append("goto ");
                int n = switchLabels.indexOf(gotoCase.Label);
                if (n == -1)
                {
                    n = switchLabels.size();
                    switchLabels.add(gotoCase.Label);
                }
                if (gotoCase.Label.Default)
                {
                    sb.append("default ").append("#").append(n).append("#");
                }
                else
                {
                    sb.append("case ").append("#").append(n).append("#");
                    if (gotoCase.Label.Name != null)
                    {
                        sb.append("\"").append(gotoCase.Label.Name).append("\"");
                    }
                    else
                    {
                        sb.append(gotoCase.Label.Value);
                    }
                }

                sb.append(";");
                break;
            }

            case Expression:
                var expression = (ExpressionStatement)statement;
                expressionToString(expression.Expression, sb, indent);
                sb.append(";");
                break;

            case If:
                var ifStatement = (IfStatement)statement;
                sb.append(indent).append("if (");
                expressionToString(ifStatement.Condition, sb, "");
                sb.append(")\n");
                statementToString(ifStatement.IfTrue, sb, indent + "    ");
                if (ifStatement.IfFalse != null)
                {
                    sb.append(indent).append("else\n");
                    statementToString(ifStatement.IfFalse, sb, indent + "    ");
                }
                break;

            case Labeled:
                var labeled = (LabeledStatement)statement;
                sb.append(indent);
                labelToString(labeled.Label, sb);
                sb.append(":\n");
                statementToString(labeled.Statement, sb, indent);
                break;

            case Return:
                sb.append(indent).append("return ");
                expressionToString(((ReturnStatement)statement).Value, sb, "");
                sb.append(";");
                break;

            case Switch:
                var switchStatement = (SwitchStatement)statement;
                sb.append(indent).append("switch (");
                expressionToString(switchStatement.Expression, sb, "");
                sb.append(") {\n");
                foreach (var section in switchStatement.Sections)
                {
                    foreach (var label in section.Labels)
                    {
                        int n = switchLabels.indexOf(label);
                        if (n == -1)
                        {
                            n = switchLabels.size();
                            switchLabels.add(label);
                        }
                        sb.append(indent);
                        if (label.Default)
                        {
                            sb.append("#").append(n).append("#").append("default:\n");
                        }
                        else
                        {
                            sb.append("#").append(n).append("#").append("case ");
                            if (label.Name != null)
                            {
                                sb.append("\"").append(label.Name).append("\"");
                            }
                            else
                            {
                                sb.append(label.Value);
                            }
                            sb.append(":\n");
                        }
                    }
                    foreach (var s in section.Statements)
                    {
                        statementToString(s, sb, indent + "    ");
                    }
                }
                sb.append(indent).append("}");
                break;

            case Synchronized:
                var synchronizedStatement = (SynchronizedStatement)statement;
                sb.append(indent).append("synchronized (");
                expressionToString(synchronizedStatement.Lock, sb, "");
                sb.append(")\n");
                statementToString(synchronizedStatement.Body, sb, indent + "    ");
                break;

            case Throw:
                sb.append(indent).append("throw ");
                expressionToString(((ThrowStatement)statement).Exception, sb, "");
                sb.append(";");
                break;

            case Try:
                var tryStatement = (TryStatement)statement;
                sb.append(indent).append("try\n");
                statementToString(tryStatement.Body, sb, indent);
                foreach (var c in tryStatement.CatchClauses)
                {
                    sb.append(indent).append("catch (");
                    variableToString(c.Variable, sb);
                    sb.append(")\n");
                    statementToString(c.Body, sb, indent);
                }
                if (tryStatement.Finally != null)
                {
                    sb.append(indent).append("finally\n");
                    statementToString(tryStatement.Finally, sb, indent);
                }
                break;

            case While:
                var whileStatement = (LoopStatement)statement;
                sb.append(indent).append("while (");
                expressionToString(whileStatement.Condition, sb, "");
                sb.append(")\n");
                statementToString(whileStatement.Body, sb, indent + "    ");
                break;

            default:
                throw new IllegalStateException("Unhandled statement " + statement.StatementKind);
            }
            sb.append("\n");
        }
示例#21
0
        public void TestRandom()
        {
            string[]      terms = new string[TestUtil.NextInt32(Random, 2, 10)];
            ISet <string> seen  = new JCG.HashSet <string>();

            while (seen.size() < terms.Length)
            {
                string token = TestUtil.RandomSimpleString(Random, 1, 5);
                if (!seen.contains(token))
                {
                    terms[seen.size()] = token;
                    seen.add(token);
                }
            }

            Analyzer a = new MockAnalyzer(Random);

            int  numDocs   = AtLeast(10);
            long totTokens = 0;

            string[][] docs = new string[numDocs][];
            for (int i = 0; i < numDocs; i++)
            {
                docs[i] = new string[AtLeast(100)];
                if (Verbose)
                {
                    Console.Write("  doc " + i + ":");
                }
                for (int j = 0; j < docs[i].Length; j++)
                {
                    docs[i][j] = GetZipfToken(terms);
                    if (Verbose)
                    {
                        Console.Write(" " + docs[i][j]);
                    }
                }
                if (Verbose)
                {
                    Console.WriteLine();
                }
                totTokens += docs[i].Length;
            }

            int grams = TestUtil.NextInt32(Random, 1, 4);

            if (Verbose)
            {
                Console.WriteLine("TEST: " + terms.Length + " terms; " + numDocs + " docs; " + grams + " grams");
            }

            // Build suggester model:
            FreeTextSuggester sug = new FreeTextSuggester(a, a, grams, (byte)0x20);

            sug.Build(new TestRandomInputEnumerator(docs));

            // Build inefficient but hopefully correct model:
            List <IDictionary <string, int?> > gramCounts = new List <IDictionary <string, int?> >(grams);

            for (int gram = 0; gram < grams; gram++)
            {
                if (Verbose)
                {
                    Console.WriteLine("TEST: build model for gram=" + gram);
                }
                IDictionary <string, int?> model = new JCG.Dictionary <string, int?>();
                gramCounts.Add(model);
                foreach (string[] doc in docs)
                {
                    for (int i = 0; i < doc.Length - gram; i++)
                    {
                        StringBuilder b = new StringBuilder();
                        for (int j = i; j <= i + gram; j++)
                        {
                            if (j > i)
                            {
                                b.append(' ');
                            }
                            b.append(doc[j]);
                        }
                        string token = b.toString();
                        if (!model.TryGetValue(token, out int?curCount) || curCount == null)
                        {
                            model.Put(token, 1);
                        }
                        else
                        {
                            model.Put(token, 1 + curCount);
                        }
                        if (Verbose)
                        {
                            Console.WriteLine("  add '" + token + "' -> count=" + (model.TryGetValue(token, out int?count) ? (count.HasValue ? count.ToString() : "null") : ""));
                        }
                    }
                }
            }

            int lookups = AtLeast(100);

            for (int iter = 0; iter < lookups; iter++)
            {
                string[] tokens = new string[TestUtil.NextInt32(Random, 1, 5)];
                for (int i = 0; i < tokens.Length; i++)
                {
                    tokens[i] = GetZipfToken(terms);
                }

                // Maybe trim last token; be sure not to create the
                // empty string:
                int trimStart;
                if (tokens.Length == 1)
                {
                    trimStart = 1;
                }
                else
                {
                    trimStart = 0;
                }
                int trimAt = TestUtil.NextInt32(Random, trimStart, tokens[tokens.Length - 1].Length);
                tokens[tokens.Length - 1] = tokens[tokens.Length - 1].Substring(0, trimAt - 0);

                int           num = TestUtil.NextInt32(Random, 1, 100);
                StringBuilder b   = new StringBuilder();
                foreach (string token in tokens)
                {
                    b.append(' ');
                    b.append(token);
                }
                string query = b.toString();
                query = query.Substring(1);

                if (Verbose)
                {
                    Console.WriteLine("\nTEST: iter=" + iter + " query='" + query + "' num=" + num);
                }

                // Expected:
                List <Lookup.LookupResult> expected = new List <Lookup.LookupResult>();
                double backoff = 1.0;
                seen = new JCG.HashSet <string>();

                if (Verbose)
                {
                    Console.WriteLine("  compute expected");
                }
                for (int i = grams - 1; i >= 0; i--)
                {
                    if (Verbose)
                    {
                        Console.WriteLine("    grams=" + i);
                    }

                    if (tokens.Length < i + 1)
                    {
                        // Don't have enough tokens to use this model
                        if (Verbose)
                        {
                            Console.WriteLine("      skip");
                        }
                        continue;
                    }

                    if (i == 0 && tokens[tokens.Length - 1].Length == 0)
                    {
                        // Never suggest unigrams from empty string:
                        if (Verbose)
                        {
                            Console.WriteLine("      skip unigram priors only");
                        }
                        continue;
                    }

                    // Build up "context" ngram:
                    b = new StringBuilder();
                    for (int j = tokens.Length - i - 1; j < tokens.Length - 1; j++)
                    {
                        b.append(' ');
                        b.append(tokens[j]);
                    }
                    string context = b.toString();
                    if (context.Length > 0)
                    {
                        context = context.Substring(1);
                    }
                    if (Verbose)
                    {
                        Console.WriteLine("      context='" + context + "'");
                    }
                    long contextCount;
                    if (context.Length == 0)
                    {
                        contextCount = totTokens;
                    }
                    else
                    {
                        //int? count = gramCounts.get(i - 1).get(context);
                        var gramCount = gramCounts[i - 1];
                        if (!gramCount.TryGetValue(context, out int?count) || count == null)
                        {
                            // We never saw this context:
                            backoff *= FreeTextSuggester.ALPHA;
                            if (Verbose)
                            {
                                Console.WriteLine("      skip: never saw context");
                            }
                            continue;
                        }
                        contextCount = count.GetValueOrDefault();
                    }
                    if (Verbose)
                    {
                        Console.WriteLine("      contextCount=" + contextCount);
                    }
                    IDictionary <string, int?> model = gramCounts[i];

                    // First pass, gather all predictions for this model:
                    if (Verbose)
                    {
                        Console.WriteLine("      find terms w/ prefix=" + tokens[tokens.Length - 1]);
                    }
                    List <Lookup.LookupResult> tmp = new List <Lookup.LookupResult>();
                    foreach (string term in terms)
                    {
                        if (term.StartsWith(tokens[tokens.Length - 1], StringComparison.Ordinal))
                        {
                            if (Verbose)
                            {
                                Console.WriteLine("        term=" + term);
                            }
                            if (seen.contains(term))
                            {
                                if (Verbose)
                                {
                                    Console.WriteLine("          skip seen");
                                }
                                continue;
                            }
                            string ngram = (context + " " + term).Trim();
                            //Integer count = model.get(ngram);
                            if (model.TryGetValue(ngram, out int?count) && count != null)
                            {
                                // LUCENENET NOTE: We need to calculate this as decimal because when using double it can sometimes
                                // return numbers that are greater than long.MaxValue, which results in a negative long number.
                                // This is also the way it is being done in the FreeTextSuggester to work around the issue.
                                Lookup.LookupResult lr = new Lookup.LookupResult(ngram, (long)(long.MaxValue * ((decimal)backoff * (decimal)count / contextCount)));
                                tmp.Add(lr);
                                if (Verbose)
                                {
                                    Console.WriteLine("      add tmp key='" + lr.Key + "' score=" + lr.Value);
                                }
                            }
                        }
                    }

                    // Second pass, trim to only top N, and fold those
                    // into overall suggestions:
                    tmp.Sort(byScoreThenKey);
                    if (tmp.size() > num)
                    {
                        //tmp.subList(num, tmp.size()).clear();
                        tmp.RemoveRange(num, tmp.size() - num); // LUCENENET: Converted end index to length
                    }
                    foreach (Lookup.LookupResult result in tmp)
                    {
                        string key = result.Key.toString();
                        int    idx = key.LastIndexOf(' ');
                        string lastToken;
                        if (idx != -1)
                        {
                            lastToken = key.Substring(idx + 1);
                        }
                        else
                        {
                            lastToken = key;
                        }
                        if (!seen.contains(lastToken))
                        {
                            seen.add(lastToken);
                            expected.Add(result);
                            if (Verbose)
                            {
                                Console.WriteLine("      keep key='" + result.Key + "' score=" + result.Value);
                            }
                        }
                    }

                    backoff *= FreeTextSuggester.ALPHA;
                }

                expected.Sort(byScoreThenKey);

                if (expected.size() > num)
                {
                    expected.RemoveRange(num, expected.size() - num); // LUCENENET: Converted end index to length
                }

                // Actual:
                IList <Lookup.LookupResult> actual = sug.DoLookup(query, num);

                if (Verbose)
                {
                    Console.WriteLine("  expected: " + expected);
                    Console.WriteLine("    actual: " + actual);
                }

                assertEquals(expected.ToString(), actual.ToString());
            }
        }
示例#22
0
        /// <summary>
        /// Encode this Exclude with elements separated by "," and Exclude.Type.ANY
        /// shown as "///".
        /// </summary>
        ///
        /// <returns>the URI string</returns>
        public String toUri()
        {
            if ((entries_.Count==0))
                return "";

            StringBuilder result = new StringBuilder();
            for (int i = 0; i < entries_.Count; ++i) {
                if (i > 0)
                    result.append(",");

                if (get(i).getType() == net.named_data.jndn.Exclude.Type.ANY)
                    result.append("*");
                else
                    get(i).getComponent().toEscapedString(result);
            }

            return result.toString();
        }
示例#23
0
        public void assertRangeQuery(NumberType?lowerType, NumberType?upperType,
                                     bool lowerInclusive, bool upperInclusive, int expectedDocCount)
        {
            StringBuilder sb = new StringBuilder();

            String lowerInclusiveStr = (lowerInclusive ? "[" : "{");
            String upperInclusiveStr = (upperInclusive ? "]" : "}");

            foreach (NumericType type in Enum.GetValues(typeof(NumericType)))
            {
                if (type == NumericType.NONE)
                {
                    continue;
                }

                String lowerStr = NumberToString(GetNumberType(lowerType, type.ToString()));
                String upperStr = NumberToString(GetNumberType(upperType, type.ToString()));

                sb.append("+").append(type.ToString()).append(':').append(lowerInclusiveStr)
                .append('"').append(lowerStr).append("\" TO \"").append(upperStr)
                .append('"').append(upperInclusiveStr).append(' ');
            }

            /*Number*/
            object lowerDateNumber = GetNumberType(lowerType, DATE_FIELD_NAME);
            /*Number*/
            object upperDateNumber = GetNumberType(upperType, DATE_FIELD_NAME);
            String lowerDateStr;
            String upperDateStr;

            if (lowerDateNumber != null)
            {
                //lowerDateStr = ESCAPER.Escape(
                //    DATE_FORMAT.format(new DateTime(lowerDateNumber.longValue())), LOCALE,
                //    EscapeQuerySyntax.Type.STRING).toString();

                lowerDateStr = ESCAPER.Escape(
                    DATE_FORMAT.Format(Convert.ToInt64(lowerDateNumber, CultureInfo.InvariantCulture)),
                    LOCALE,
                    EscapeQuerySyntaxType.STRING).toString();
            }
            else
            {
                lowerDateStr = "*";
            }

            if (upperDateNumber != null)
            {
                //upperDateStr = ESCAPER.Escape(
                //      DATE_FORMAT.format(new DateTime(upperDateNumber.longValue())), LOCALE,
                //      EscapeQuerySyntax.Type.STRING).toString();

                upperDateStr = ESCAPER.Escape(
                    DATE_FORMAT.Format(Convert.ToInt64(upperDateNumber, CultureInfo.InvariantCulture)),
                    LOCALE,
                    EscapeQuerySyntaxType.STRING).toString();
            }
            else
            {
                upperDateStr = "*";
            }

            sb.append("+").append(DATE_FIELD_NAME).append(':')
            .append(lowerInclusiveStr).append('"').append(lowerDateStr).append(
                "\" TO \"").append(upperDateStr).append('"').append(
                upperInclusiveStr);


            TestQuery(sb.toString(), expectedDocCount);
        }
示例#24
0
        /// <summary>
        /// Encode the name according to the "NDN URI Scheme".  If there are interest
        /// selectors, append "?" and added the selectors as a query string.  For
        /// example "/test/name?ndn.ChildSelector=1".
        /// </summary>
        ///
        /// <returns>The URI string.</returns>
        /// @note This is an experimental feature.  See the API docs for more detail at
        /// http://named-data.net/doc/ndn-ccl-api/interest.html#interest-touri-method .
        public String toUri()
        {
            StringBuilder selectors = new StringBuilder();

            if (minSuffixComponents_ >= 0)
                selectors.append("&ndn.MinSuffixComponents=").append(
                        minSuffixComponents_);
            if (maxSuffixComponents_ >= 0)
                selectors.append("&ndn.MaxSuffixComponents=").append(
                        maxSuffixComponents_);
            if (childSelector_ >= 0)
                selectors.append("&ndn.ChildSelector=").append(childSelector_);
            selectors.append("&ndn.MustBeFresh=").append((mustBeFresh_) ? 1 : 0);
            if (interestLifetimeMilliseconds_ >= 0)
                selectors.append("&ndn.InterestLifetime=").append(
                        (long) Math.Round(interestLifetimeMilliseconds_,MidpointRounding.AwayFromZero));
            if (nonce_.size() > 0) {
                selectors.append("&ndn.Nonce=");
                net.named_data.jndn.Name.toEscapedString(nonce_.buf(), selectors);
            }
            if (getExclude().size() > 0)
                selectors.append("&ndn.Exclude=").append(getExclude().toUri());

            StringBuilder result = new StringBuilder();

            result.append(getName().toUri());
            String selectorsString = selectors.toString();
            if (selectorsString.Length > 0)
                // Replace the first & with ?.
                result.append("?").append(selectorsString.Substring(1));

            return result.toString();
        }
示例#25
0
 /**
  * Returns the state of this tokenizer in a readable format.
  *
  * @return the current state of this tokenizer.
  */
 public override String ToString()
 {
     // Values determined through experimentation
     StringBuilder result = new StringBuilder();
     result.append("Token["); //$NON-NLS-1$
     switch (ttype) {
         case TT_EOF:
             result.append("EOF"); //$NON-NLS-1$
             break;
         case TT_EOL:
             result.append("EOL"); //$NON-NLS-1$
             break;
         case TT_NUMBER:
             result.append("n="); //$NON-NLS-1$
             result.append(nval);
             break;
         case TT_WORD:
             result.append(sval);
             break;
         default:
             if (ttype == TT_UNKNOWN || tokenTypes[ttype] == TOKEN_QUOTE) {
                 result.append(sval);
             } else {
                 result.append('\'');
                 result.append((char) ttype);
                 result.append('\'');
             }
             break;
     }
     result.append("], line "); //$NON-NLS-1$
     result.append(lineNumber);
     return result.toString();
 }
示例#26
0
 /**
  * Other characters, which are Unicode chars that are not US-ASCII, and are
  * not ISO Control or are not ISO Space chars are not preserved. They are
  * converted into their hexidecimal value prepended by '%'.
  * <p>
  * For example: Euro currency symbol -> "%E2%82%AC".
  * <p>
  * Called from URI.toASCIIString()
  *
  * @param s
  *            java.lang.String the string to be converted
  * @return java.lang.String the converted string
  */
 static String encodeOthers(String s)
 {
     //throws UnsupportedEncodingException {
     StringBuilder buf = new StringBuilder();
     for (int i = 0; i < s.length(); i++) {
         char ch = s.charAt(i);
         if (ch <= 127) {
             buf.append(ch);
         } else {
             byte[] bytes = new String(new char[] { ch }).getBytes(encoding);
             for (int j = 0; j < bytes.Length; j++) {
                 buf.append('%');
                 buf.append(digits.charAt((bytes[j] & 0xf0) >> 4));
                 buf.append(digits.charAt(bytes[j] & 0xf));
             }
         }
     }
     return buf.toString();
 }
示例#27
0
        public void TestRandom()
        {
            int numberOfRuns = TestUtil.NextInt32(Random, 3, 6);

            for (int iter = 0; iter < numberOfRuns; iter++)
            {
                if (Verbose)
                {
                    Console.WriteLine(string.Format("TEST: iter={0} total={1}", iter, numberOfRuns));
                }

                int numDocs   = TestUtil.NextInt32(Random, 100, 1000) * RandomMultiplier;
                int numGroups = TestUtil.NextInt32(Random, 1, numDocs);

                if (Verbose)
                {
                    Console.WriteLine("TEST: numDocs=" + numDocs + " numGroups=" + numGroups);
                }

                List <BytesRef> groups = new List <BytesRef>();
                for (int i = 0; i < numGroups; i++)
                {
                    string randomValue;
                    do
                    {
                        // B/c of DV based impl we can't see the difference between an empty string and a null value.
                        // For that reason we don't generate empty string groups.
                        randomValue = TestUtil.RandomRealisticUnicodeString(Random);
                    } while ("".Equals(randomValue, StringComparison.Ordinal));
                    groups.Add(new BytesRef(randomValue));
                }
                string[] contentStrings = new string[TestUtil.NextInt32(Random, 2, 20)];
                if (Verbose)
                {
                    Console.WriteLine("TEST: create fake content");
                }
                for (int contentIDX = 0; contentIDX < contentStrings.Length; contentIDX++)
                {
                    StringBuilder sb = new StringBuilder();
                    sb.append("real").append(Random.nextInt(3)).append(' ');
                    int fakeCount = Random.nextInt(10);
                    for (int fakeIDX = 0; fakeIDX < fakeCount; fakeIDX++)
                    {
                        sb.append("fake ");
                    }
                    contentStrings[contentIDX] = sb.toString();
                    if (Verbose)
                    {
                        Console.WriteLine("  content=" + sb.toString());
                    }
                }

                Directory         dir = NewDirectory();
                RandomIndexWriter w   = new RandomIndexWriter(
                    Random,
                    dir,
                    NewIndexWriterConfig(TEST_VERSION_CURRENT,
                                         new MockAnalyzer(Random)));
                bool          preFlex   = "Lucene3x".Equals(w.IndexWriter.Config.Codec.Name, StringComparison.Ordinal);
                bool          canUseIDV = !preFlex;
                DocValuesType valueType = vts[Random.nextInt(vts.Length)];

                Document doc        = new Document();
                Document docNoGroup = new Document();
                Field    group      = NewStringField("group", "", Field.Store.NO);
                doc.Add(group);
                Field valuesField = null;
                if (canUseIDV)
                {
                    switch (valueType)
                    {
                    case DocValuesType.BINARY:
                        valuesField = new BinaryDocValuesField("group_dv", new BytesRef());
                        break;

                    case DocValuesType.SORTED:
                        valuesField = new SortedDocValuesField("group_dv", new BytesRef());
                        break;

                    default:
                        fail("unhandled type");
                        break;
                    }
                    doc.Add(valuesField);
                }
                Field sort1 = NewStringField("sort1", "", Field.Store.NO);
                doc.Add(sort1);
                docNoGroup.Add(sort1);
                Field sort2 = NewStringField("sort2", "", Field.Store.NO);
                doc.Add(sort2);
                docNoGroup.Add(sort2);
                Field sort3 = NewStringField("sort3", "", Field.Store.NO);
                doc.Add(sort3);
                docNoGroup.Add(sort3);
                Field content = NewTextField("content", "", Field.Store.NO);
                doc.Add(content);
                docNoGroup.Add(content);
                Int32Field id = new Int32Field("id", 0, Field.Store.NO);
                doc.Add(id);
                docNoGroup.Add(id);
                GroupDoc[] groupDocs = new GroupDoc[numDocs];
                for (int i = 0; i < numDocs; i++)
                {
                    BytesRef groupValue;
                    if (Random.nextInt(24) == 17)
                    {
                        // So we test the "doc doesn't have the group'd
                        // field" case:
                        groupValue = null;
                    }
                    else
                    {
                        groupValue = groups[Random.nextInt(groups.size())];
                    }

                    GroupDoc groupDoc = new GroupDoc(
                        i,
                        groupValue,
                        groups[Random.nextInt(groups.size())],
                        groups[Random.nextInt(groups.size())],
                        new BytesRef(string.Format(CultureInfo.InvariantCulture, "{0:D5}", i)),
                        contentStrings[Random.nextInt(contentStrings.Length)]
                        );

                    if (Verbose)
                    {
                        Console.WriteLine("  doc content=" + groupDoc.content + " id=" + i + " group=" + (groupDoc.group == null ? "null" : groupDoc.group.Utf8ToString()) + " sort1=" + groupDoc.sort1.Utf8ToString() + " sort2=" + groupDoc.sort2.Utf8ToString() + " sort3=" + groupDoc.sort3.Utf8ToString());
                    }

                    groupDocs[i] = groupDoc;
                    if (groupDoc.group != null)
                    {
                        group.SetStringValue(groupDoc.group.Utf8ToString());
                        if (canUseIDV)
                        {
                            valuesField.SetBytesValue(new BytesRef(groupDoc.group.Utf8ToString()));
                        }
                    }
                    sort1.SetStringValue(groupDoc.sort1.Utf8ToString());
                    sort2.SetStringValue(groupDoc.sort2.Utf8ToString());
                    sort3.SetStringValue(groupDoc.sort3.Utf8ToString());
                    content.SetStringValue(groupDoc.content);
                    id.SetInt32Value(groupDoc.id);
                    if (groupDoc.group == null)
                    {
                        w.AddDocument(docNoGroup);
                    }
                    else
                    {
                        w.AddDocument(doc);
                    }
                }

                DirectoryReader r = w.GetReader();
                w.Dispose();

                // NOTE: intentional but temporary field cache insanity!
                FieldCache.Int32s docIdToFieldId = FieldCache.DEFAULT.GetInt32s(SlowCompositeReaderWrapper.Wrap(r), "id", false);
                int[]             fieldIdToDocID = new int[numDocs];
                for (int i = 0; i < numDocs; i++)
                {
                    int fieldId = docIdToFieldId.Get(i);
                    fieldIdToDocID[fieldId] = i;
                }

                try
                {
                    IndexSearcher s = NewSearcher(r);
                    if (typeof(SlowCompositeReaderWrapper).IsAssignableFrom(s.IndexReader.GetType()))
                    {
                        canUseIDV = false;
                    }
                    else
                    {
                        canUseIDV = !preFlex;
                    }

                    for (int contentID = 0; contentID < 3; contentID++)
                    {
                        ScoreDoc[] hits = s.Search(new TermQuery(new Term("content", "real" + contentID)), numDocs).ScoreDocs;
                        foreach (ScoreDoc hit in hits)
                        {
                            GroupDoc gd = groupDocs[docIdToFieldId.Get(hit.Doc)];
                            assertTrue(gd.score == 0.0);
                            gd.score = hit.Score;
                            int docId = gd.id;
                            assertEquals(docId, docIdToFieldId.Get(hit.Doc));
                        }
                    }

                    foreach (GroupDoc gd in groupDocs)
                    {
                        assertTrue(gd.score != 0.0);
                    }

                    for (int searchIter = 0; searchIter < 100; searchIter++)
                    {
                        if (Verbose)
                        {
                            Console.WriteLine("TEST: searchIter=" + searchIter);
                        }

                        string searchTerm      = "real" + Random.nextInt(3);
                        bool   sortByScoreOnly = Random.nextBoolean();
                        Sort   sortWithinGroup = GetRandomSort(sortByScoreOnly);
                        AbstractAllGroupHeadsCollector allGroupHeadsCollector = CreateRandomCollector("group", sortWithinGroup, canUseIDV, valueType);
                        s.Search(new TermQuery(new Term("content", searchTerm)), allGroupHeadsCollector);
                        int[] expectedGroupHeads = CreateExpectedGroupHeads(searchTerm, groupDocs, sortWithinGroup, sortByScoreOnly, fieldIdToDocID);
                        int[] actualGroupHeads   = allGroupHeadsCollector.RetrieveGroupHeads();
                        // The actual group heads contains Lucene ids. Need to change them into our id value.
                        for (int i = 0; i < actualGroupHeads.Length; i++)
                        {
                            actualGroupHeads[i] = docIdToFieldId.Get(actualGroupHeads[i]);
                        }
                        // Allows us the easily iterate and assert the actual and expected results.
                        Array.Sort(expectedGroupHeads);
                        Array.Sort(actualGroupHeads);

                        if (Verbose)
                        {
                            Console.WriteLine("Collector: " + allGroupHeadsCollector.GetType().Name);
                            Console.WriteLine("Sort within group: " + sortWithinGroup);
                            Console.WriteLine("Num group: " + numGroups);
                            Console.WriteLine("Num doc: " + numDocs);
                            Console.WriteLine("\n=== Expected: \n");
                            foreach (int expectedDocId in expectedGroupHeads)
                            {
                                GroupDoc expectedGroupDoc = groupDocs[expectedDocId];
                                string   expectedGroup    = expectedGroupDoc.group == null ? null : expectedGroupDoc.group.Utf8ToString();
                                Console.WriteLine(
                                    string.Format(CultureInfo.InvariantCulture,
                                                  "Group:{0,10} score{1:0.0#######,5} Sort1:{2,10} Sort2:{3,10} Sort3:{4,10} doc:{5,10}",
                                                  expectedGroup, expectedGroupDoc.score, expectedGroupDoc.sort1.Utf8ToString(),
                                                  expectedGroupDoc.sort2.Utf8ToString(), expectedGroupDoc.sort3.Utf8ToString(), expectedDocId)
                                    );
                            }
                            Console.WriteLine("\n=== Actual: \n");
                            foreach (int actualDocId in actualGroupHeads)
                            {
                                GroupDoc actualGroupDoc = groupDocs[actualDocId];
                                string   actualGroup    = actualGroupDoc.group == null ? null : actualGroupDoc.group.Utf8ToString();
                                Console.WriteLine(
                                    string.Format(CultureInfo.InvariantCulture,
                                                  "Group:{0,10} score{1:0.0#######,5} Sort1:{2,10} Sort2:{3,10} Sort3:{4,10} doc:{5,10}",
                                                  actualGroup, actualGroupDoc.score, actualGroupDoc.sort1.Utf8ToString(),
                                                  actualGroupDoc.sort2.Utf8ToString(), actualGroupDoc.sort3.Utf8ToString(), actualDocId)
                                    );
                            }
                            Console.WriteLine("\n===================================================================================");
                        }

                        assertArrayEquals(expectedGroupHeads, actualGroupHeads);
                    }
                }
                finally
                {
                    QueryUtils.PurgeFieldCache(r);
                }

                r.Dispose();
                dir.Dispose();
            }
        }
示例#28
0
        /**
         * Parses the next token from this tokenizer's source stream or reader. The
         * type of the token is stored in the {@code ttype} field, additional
         * information may be stored in the {@code nval} or {@code sval} fields.
         *
         * @return the value of {@code ttype}.
         * @throws IOException
         *             if an I/O error occurs while parsing the next token.
         */
        public virtual int nextToken()
        {
            // throws IOException {
            if (pushBackToken) {
                pushBackToken = false;
                if (ttype != TT_UNKNOWN) {
                    return ttype;
                }
            }
            sval = null; // Always reset sval to null
            int currentChar = peekChar == -2 ? read() : peekChar;

            if (lastCr && currentChar == '\n') {
                lastCr = false;
                currentChar = read();
            }
            if (currentChar == -1) {
                return (ttype = TT_EOF);
            }

            byte currentType = currentChar > 255 ? TOKEN_WORD
                    : tokenTypes[currentChar];
            while ((currentType & TOKEN_WHITE) != 0) {
                /**
                 * Skip over white space until we hit a new line or a real token
                 */
                if (currentChar == '\r') {
                    lineNumber++;
                    if (isEOLSignificant) {
                        lastCr = true;
                        peekChar = -2;
                        return (ttype = TT_EOL);
                    }
                    if ((currentChar = read()) == '\n') {
                        currentChar = read();
                    }
                } else if (currentChar == '\n') {
                    lineNumber++;
                    if (isEOLSignificant) {
                        peekChar = -2;
                        return (ttype = TT_EOL);
                    }
                    currentChar = read();
                } else {
                    // Advance over this white space character and try again.
                    currentChar = read();
                }
                if (currentChar == -1) {
                    return (ttype = TT_EOF);
                }
                currentType = currentChar > 255 ? TOKEN_WORD
                        : tokenTypes[currentChar];
            }

            /**
             * Check for digits before checking for words since digits can be
             * contained within words.
             */
            if ((currentType & TOKEN_DIGIT) != 0) {
                StringBuilder digits = new StringBuilder(20);
                bool haveDecimal = false, checkJustNegative = currentChar == '-';
                while (true) {
                    if (currentChar == '.') {
                        haveDecimal = true;
                    }
                    digits.append((char) currentChar);
                    currentChar = read();
                    if ((currentChar < '0' || currentChar > '9')
                            && (haveDecimal || currentChar != '.')) {
                        break;
                    }
                }
                peekChar = currentChar;
                if (checkJustNegative && digits.Length == 1) {
                    // Didn't get any other digits other than '-'
                    return (ttype = '-');
                }
                try {
                    nval = java.lang.Double.valueOf(digits.toString()).doubleValue();
                } catch (java.lang.NumberFormatException e) {
                    // Unsure what to do, will write test.
                    nval = 0;
                }
                return (ttype = TT_NUMBER);
            }
            // Check for words
            if ((currentType & TOKEN_WORD) != 0) {
                StringBuilder word = new StringBuilder(20);
                while (true) {
                    word.append((char) currentChar);
                    currentChar = read();
                    if (currentChar == -1
                            || (currentChar < 256 && (tokenTypes[currentChar] & (TOKEN_WORD | TOKEN_DIGIT)) == 0)) {
                        break;
                    }
                }
                peekChar = currentChar;
                sval = forceLowercase ? word.toString().ToLower() : word
                        .toString();
                return (ttype = TT_WORD);
            }
            // Check for quoted character
            if (currentType == TOKEN_QUOTE) {
                int matchQuote = currentChar;
                StringBuilder quoteString = new StringBuilder();
                int peekOne = read();
                while (peekOne >= 0 && peekOne != matchQuote && peekOne != '\r'
                        && peekOne != '\n') {
                    bool readPeek = true;
                    if (peekOne == '\\') {
                        int c1 = read();
                        // Check for quoted octal IE: \377
                        if (c1 <= '7' && c1 >= '0') {
                            int digitValue = c1 - '0';
                            c1 = read();
                            if (c1 > '7' || c1 < '0') {
                                readPeek = false;
                            } else {
                                digitValue = digitValue * 8 + (c1 - '0');
                                c1 = read();
                                // limit the digit value to a byte
                                if (digitValue > 037 || c1 > '7' || c1 < '0') {
                                    readPeek = false;
                                } else {
                                    digitValue = digitValue * 8 + (c1 - '0');
                                }
                            }
                            if (!readPeek) {
                                // We've consumed one to many
                                quoteString.append((char) digitValue);
                                peekOne = c1;
                            } else {
                                peekOne = digitValue;
                            }
                        } else {
                            switch (c1) {
                                case 'a':
                                    peekOne = 0x7;
                                    break;
                                case 'b':
                                    peekOne = 0x8;
                                    break;
                                case 'f':
                                    peekOne = 0xc;
                                    break;
                                case 'n':
                                    peekOne = 0xA;
                                    break;
                                case 'r':
                                    peekOne = 0xD;
                                    break;
                                case 't':
                                    peekOne = 0x9;
                                    break;
                                case 'v':
                                    peekOne = 0xB;
                                    break;
                                default:
                                    peekOne = c1;
                                    break;
                            }
                        }
                    }
                    if (readPeek) {
                        quoteString.append((char) peekOne);
                        peekOne = read();
                    }
                }
                if (peekOne == matchQuote) {
                    peekOne = read();
                }
                peekChar = peekOne;
                ttype = matchQuote;
                sval = quoteString.toString();
                return ttype;
            }
            // Do comments, both "//" and "/*stuff*/"
            if (currentChar == '/' && (slashSlashCommentsJ || slashStarCommentsJ)) {
                if ((currentChar = read()) == '*' && slashStarCommentsJ) {
                    int peekOne = read();
                    while (true) {
                        currentChar = peekOne;
                        peekOne = read();
                        if (currentChar == -1) {
                            peekChar = -1;
                            return (ttype = TT_EOF);
                        }
                        if (currentChar == '\r') {
                            if (peekOne == '\n') {
                                peekOne = read();
                            }
                            lineNumber++;
                        } else if (currentChar == '\n') {
                            lineNumber++;
                        } else if (currentChar == '*' && peekOne == '/') {
                            peekChar = read();
                            return nextToken();
                        }
                    }
                } else if (currentChar == '/' && slashSlashCommentsJ) {
                    // Skip to EOF or new line then return the next token
                    while ((currentChar = read()) >= 0 && currentChar != '\r'
                            && currentChar != '\n') {
                        // Intentionally empty
                    }
                    peekChar = currentChar;
                    return nextToken();
                } else if (currentType != TOKEN_COMMENT) {
                    // Was just a slash by itself
                    peekChar = currentChar;
                    return (ttype = '/');
                }
            }
            // Check for comment character
            if (currentType == TOKEN_COMMENT) {
                // Skip to EOF or new line then return the next token
                while ((currentChar = read()) >= 0 && currentChar != '\r'
                        && currentChar != '\n') {
                    // Intentionally empty
                }
                peekChar = currentChar;
                return nextToken();
            }

            peekChar = read();
            return (ttype = currentChar);
        }
示例#29
0
 protected internal virtual String convertPattern(String template, String fromChars, String toChars,
         bool check)
 {
     if (!check && fromChars.equals(toChars)) {
         return template;
     }
     bool quote = false;
     StringBuilder output = new StringBuilder();
     int length = template.length();
     for (int i = 0; i < length; i++) {
         int index;
         char next = template.charAt(i);
         if (next == '\'') {
             quote = !quote;
         }
         if (!quote && (index = fromChars.indexOf(next)) != -1) {
             output.append(toChars.charAt(index));
         } else if (check
                 && !quote
                 && ((next >= 'a' && next <= 'z') || (next >= 'A' && next <= 'Z'))) {
             // text.05=Invalid pattern char {0} in {1}
             throw new java.lang.IllegalArgumentException("Invalid pattern char "+next+" in "+ template); //$NON-NLS-1$
         } else {
             output.append(next);
         }
     }
     if (quote) {
         // text.04=Unterminated quote
         throw new java.lang.IllegalArgumentException("Unterminated quote"); //$NON-NLS-1$
     }
     return output.toString();
 }
 public String toString() {
   StringBuilder buffer = new StringBuilder();
   buffer.append("mockFs{files:[");
   for(int i=0; i < files.size(); ++i) {
     if (i != 0) {
       buffer.append(", ");
     }
     buffer.append(files.get(i));
   }
   buffer.append("]}");
   return buffer.toString();
 }
示例#31
0
        } // constructor

        public override INLGElement realise(INLGElement element)
        {
            // realise a single element
            INLGElement realisedComponent = null;
            var         realisation       = new StringBuilder();

            if (element != null)
            {
                var category   = element.getCategory();
                var components = element.getChildren();

                //NB: The order of the if-statements below is important!

                // check if this is a canned text first
                if (element is StringElement)
                {
                    realisation.append(element.getRealisation());
                }
                else if (category is IDocumentCategory)
                {
                    // && element instanceof DocumentElement

                    switch ((DocumentCategoryEnum)category.enumType)
                    {
                    case DocumentCategoryEnum.DOCUMENT:
                        var title = element is DocumentElement ? ((DocumentElement)element).getTitle() : null;
                        realisation.append("<h1>" + title + "</h1>");

                        foreach (var eachComponent in components)
                        {
                            realisedComponent = realise(eachComponent);
                            if (realisedComponent != null)
                            {
                                realisation.append(realisedComponent.getRealisation());
                            }
                        }

                        break;

                    case DocumentCategoryEnum.SECTION:
                        title = element is DocumentElement ? ((DocumentElement)element).getTitle() : null;

                        if (title != null)
                        {
                            var sectionTitle = ((DocumentElement)element).getTitle();
                            realisation.append("<h2>" + sectionTitle + "</h2>");
                        }

                        foreach (var eachComponent in components)
                        {
                            realisedComponent = realise(eachComponent);
                            if (realisedComponent != null)
                            {
                                realisation.append(realisedComponent.getRealisation());
                            }
                        }
                        break;

                    case DocumentCategoryEnum.LIST:
                        realisation.append("<ul>");
                        foreach (var eachComponent in components)
                        {
                            realisedComponent = realise(eachComponent);
                            if (realisedComponent != null)
                            {
                                realisation.append(realisedComponent.getRealisation());
                            }
                        }
                        realisation.append("</ul>");
                        break;

                    case DocumentCategoryEnum.ENUMERATED_LIST:
                        realisation.append("<ol>");
                        foreach (var eachComponent in components)
                        {
                            realisedComponent = realise(eachComponent);
                            if (realisedComponent != null)
                            {
                                realisation.append(realisedComponent.getRealisation());
                            }
                        }
                        realisation.append("</ol>");
                        break;

                    case DocumentCategoryEnum.PARAGRAPH:
                        if (null != components && 0 < components.size())
                        {
                            realisedComponent = realise(components.get(0));
                            if (realisedComponent != null)
                            {
                                realisation.append("<p>");
                                realisation.append(realisedComponent.getRealisation());
                            }
                            for (var i = 1; i < components.size(); i++)
                            {
                                if (realisedComponent != null)
                                {
                                    realisation.append(" ");
                                }
                                realisedComponent = realise(components.get(i));
                                if (realisedComponent != null)
                                {
                                    realisation.append(realisedComponent.getRealisation());
                                }
                            }
                            realisation.append("</p>");
                        }

                        break;

                    case DocumentCategoryEnum.SENTENCE:
                        realisation.append(element.getRealisation());
                        break;

                    case DocumentCategoryEnum.LIST_ITEM:
                        realisation.append("<li>");

                        foreach (var eachComponent in components)
                        {
                            realisedComponent = realise(eachComponent);

                            if (realisedComponent != null)
                            {
                                realisation.append(realisedComponent.getRealisation());

                                if (components.indexOf(eachComponent) < components.size() - 1)
                                {
                                    realisation.append(' ');
                                }
                            }
                        }
                        realisation.append("</li>");

                        break;
                    }

                    // also need to check if element is a listelement (items can
                    // have embedded lists post-orthography) or a coordinate
                }
                else if (element is ListElement || element is CoordinatedPhraseElement)
                {
                    foreach (var eachComponent in components)
                    {
                        realisedComponent = realise(eachComponent);
                        if (realisedComponent != null)
                        {
                            realisation.append(realisedComponent.getRealisation()).append(' ');
                        }
                    }
                }
            }

            return(new StringElement(realisation.ToString()));
        } // realise ~ single element
	    /**
	     * Parses <code>source</code> for special double values.  These values
	     * include Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY.
	     *
	     * @param source the string to parse
	     * @param value the special value to parse.
	     * @param pos input/ouput parsing parameter.
	     * @return the special number.
	     */
	    private static Number parseNumber(String source, double value,
	                                      ParsePosition pos) {
	        Number ret = null;
	
	        StringBuilder sb = new StringBuilder();
	        sb.append('(');
	        sb.append(value);
	        sb.append(')');
	
	        int n = sb.length();
	        int startIndex = pos.getIndex();
	        int endIndex = startIndex + n;
	        if (endIndex < source.length()) {
	            if (source.substring(startIndex, endIndex).compareTo(sb.toString()) == 0) {
	                ret = Double.valueOf(value);
	                pos.setIndex(endIndex);
	            }
	        }
	
	        return ret;
	    }
        /**
         * Loop to encode a data packet nIterations times.
         * @param nIterations The number of iterations.
         * @param useComplex If true, use a large name, large content and all fields.
         * If false, use a small name, small content
         * and only required fields.
         * @param useCrypto If true, sign the data packet.  If false, use a blank
         * signature.
         * @param keyType KeyType.RSA or EC, used if useCrypto is true.
         * @param encoding Set encoding[0] to the wire encoding.
         * @return The number of seconds for all iterations.
         */
        private static double benchmarkEncodeDataSeconds(int nIterations, bool useComplex, bool useCrypto, KeyType keyType,
        Blob[] encoding)
        {
            Name name;
              Blob content;
              if (useComplex) {
            // Use a large name and content.
            name = new Name
              ("/ndn/ucla.edu/apps/lwndn-test/numbers.txt/%FD%05%05%E8%0C%CE%1D/%00");

            StringBuilder contentStream = new StringBuilder();
            int count = 1;
            contentStream.append(count++);
            while (contentStream.toString().Length < 1115)
              contentStream.append(" ").append(count++);
            content = new Blob(contentStream.toString());
              }
              else {
            // Use a small name and content.
            name = new Name("/test");
            content = new Blob("abc");
              }
              Name.Component finalBlockId =
            new Name.Component(new Blob(new byte[] { (byte)0 }));

              // Initialize the KeyChain storage in case useCrypto is true.
              MemoryIdentityStorage identityStorage = new MemoryIdentityStorage();
              MemoryPrivateKeyStorage privateKeyStorage = new MemoryPrivateKeyStorage();
              KeyChain keyChain = new KeyChain
            (new IdentityManager(identityStorage, privateKeyStorage),
              new SelfVerifyPolicyManager(identityStorage));
              Name keyName = new Name("/testname/DSK-123");
              Name certificateName = keyName.getSubName(0, keyName.size() - 1).append
            ("KEY").append(keyName.get(-1)).append("ID-CERT").append("0");
              privateKeyStorage.setKeyPairForKeyName
              (keyName, KeyType.RSA, new ByteBuffer(DEFAULT_RSA_PUBLIC_KEY_DER),
            new ByteBuffer(DEFAULT_RSA_PRIVATE_KEY_DER));

              Blob signatureBits = new Blob(new byte[256]);
              Blob emptyBlob = new Blob(new byte[0]);

              double start = getNowSeconds();
              for (int i = 0; i < nIterations; ++i) {
            Data data = new Data(name);
            data.setContent(content);
            if (useComplex) {
              data.getMetaInfo().setFreshnessPeriod(30000);
              data.getMetaInfo().setFinalBlockId(finalBlockId);
            }

            if (useCrypto)
              // This sets the signature fields.
              keyChain.sign(data, certificateName);
            else {
              // Imitate IdentityManager.signByCertificate to set up the signature
              //   fields, but don't sign.
              KeyLocator keyLocator = new KeyLocator();
              keyLocator.setType(KeyLocatorType.KEYNAME);
              keyLocator.setKeyName(certificateName);
              Sha256WithRsaSignature sha256Signature =
            (Sha256WithRsaSignature)data.getSignature();
              sha256Signature.setKeyLocator(keyLocator);
              sha256Signature.setSignature(signatureBits);
            }

            encoding[0] = data.wireEncode();
              }
              double finish = getNowSeconds();

              return finish - start;
        }
示例#34
0
	private void mHereDocContents()
	{
		EnterRule_HereDocContents();
		EnterRule("HereDocContents", 121);
		TraceIn("HereDocContents", 121);
		try
		{
			// C:\\Users\\exKAZUu\\Projects\\Code2Xml\\Code2Xml.Core\\Generators\\ANTLRv3\\Php\\Php.g:736:2: ()
			DebugEnterAlt(1);
			// C:\\Users\\exKAZUu\\Projects\\Code2Xml\\Code2Xml.Core\\Generators\\ANTLRv3\\Php\\Php.g:736:4: 
			{
			DebugLocation(736, 4);
			if (state.backtracking == 0)
			{

						// Please see also Code2Xml.Languages.ANTLRv3.Core.ExtensionForParser
						var sb = new StringBuilder();
						while (input.LA(1) != '\n') {
							sb.append((char)input.LA(1));
							input.consume();
						}
						input.consume();
						var hereDocName = sb.toString().TrimEnd();
						var hdnl = hereDocName.length();
						while (true) {
							var matchEnd = true;
							for (int i = 0; i < hdnl; i++) {
								if (input.LA(1) != hereDocName.charAt(i)) {
									matchEnd = false;
									break;
								}
								input.consume();
							}
							if (matchEnd == false) {
								while (input.LA(1) != '\n') {
									input.consume();
								}
								input.consume();
							} else {
								break;
							}
						}
					
			}

			}

		}
		finally
		{
			TraceOut("HereDocContents", 121);
			LeaveRule("HereDocContents", 121);
			LeaveRule_HereDocContents();
		}
	}
示例#35
0
        private int nextToken()
        {
            if (_peekToken > 0)
            {
                int token = _peekToken;
                _peekToken = 0;
                return(token);
            }

            while (true)
            {
                skipSpaces();

                int ch = read();

                if (ch < 0)
                {
                    return(-1);
                }
                else if (ch == '-')
                {
                    return('-');
                }
                else if (ch == '+')
                {
                    return('+');
                }
                else if (ch == ':')
                {
                    return(':');
                }
                else if (ch == '.')
                {
                    return('.');
                }
                else if (ch == '/')
                {
                    return('/');
                }
                else if (ch == '@')
                {
                    return('@');
                }
                else if ('0' <= ch && ch <= '9')
                {
                    int value  = 0;
                    int digits = 0;

                    for (; '0' <= ch && ch <= '9'; ch = read())
                    {
                        digits++;
                        value = 10 * value + ch - '0';
                    }

                    _value  = value;
                    _digits = digits;

                    unread();

                    return(INT);
                }
                else if ('a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z')
                {
                    _sb.setLength(0);

                    for (;
                         'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '.';
                         ch = read())
                    {
                        _sb.append(Character.toLowerCase((char)ch));
                    }

                    unread();

                    string s = _sb.ToString();

                    return(parseString(s));
                }
                else
                {
                    // skip
                }
            }
        }
 /// <summary>
 /// This method is used to convert the provided value into an XML
 /// usable format. This is used in the serialization process when
 /// there is a need to convert a field value in to a string so
 /// that that value can be written as a valid XML entity.
 /// </summary>
 /// <param name="value">
 /// this is the value to be converted to a string
 /// </param>
 /// <returns>
 /// this is the string representation of the given value
 /// </returns>
 public String Write(Object value, int length) {
    StringBuilder text = new StringBuilder(length);
    for(int i = 0; i < length; i++) {
       Object entry = Array.get(value, i);
       if(entry != null) {
          text.append(entry);
       }
    }
    return text.toString();
 }
示例#37
0
 public String readLine()
 {
     //throws IOException {
     StringBuilder line = new StringBuilder(80); // Typical line length
     bool foundTerminator = false;
     while (true) {
         int nextByte = inJ.read();
         switch (nextByte) {
             case -1:
                 if (line.Length == 0 && !foundTerminator) {
                     return null;
                 }
                 return line.toString();
             case (byte) '\r':
                 if (foundTerminator) {
                     ((PushbackInputStream) inJ).unread(nextByte);
                     return line.toString();
                 }
                 foundTerminator = true;
                 /* Have to be able to peek ahead one byte */
                 if (!(inJ is PushbackInputStream)) { //think Bug in Harmony, not test on class but on type required
                     inJ = new PushbackInputStream(inJ);
                 }
                 break;
             case (byte) '\n':
                 return line.toString();
             default:
                 if (foundTerminator) {
                     ((PushbackInputStream) inJ).unread(nextByte);
                     return line.toString();
                 }
                 line.append((char) nextByte);
                 break;
         }
     }
 }
 /**
  * Makes a bunch of single-char tokens (the max # unique terms will at most be 26).
  * puts the # unique terms into expected, to be checked against the norm.
  */
 private string AddValue()
 {
     StringBuilder sb = new StringBuilder();
     HashSet<string> terms = new HashSet<string>();
     int num = TestUtil.NextInt(Random(), 0, 255);
     for (int i = 0; i < num; i++)
     {
         sb.append(' ');
         char term = (char)TestUtil.NextInt(Random(), 'a', 'z');
         sb.append(term);
         terms.add("" + term);
     }
     expected.Add(terms.size());
     return sb.toString();
 }
示例#39
0
        /// <summary>
        /// Encode this name as a URI according to the NDN URI Scheme.
        /// </summary>
        ///
        /// <param name="includeScheme">which is normally the case where toUri() is used for display.</param>
        /// <returns>The URI string.</returns>
        public String toUri(bool includeScheme)
        {
            if ((components_.Count==0))
                return (includeScheme) ? "ndn:/" : "/";

            StringBuilder result = new StringBuilder();
            if (includeScheme)
                result.append("ndn:");
            for (int i = 0; i < components_.Count; ++i) {
                result.append("/");
                get(i).toEscapedString(result);
            }

            return result.toString();
        }
示例#40
0
 /// <summary>
 /// Write the value to result, escaping characters according to the NDN URI
 /// Scheme.
 /// This also adds "..." to a value with zero or more ".".
 /// This does not add a type code prefix such as "sha256digest=".
 /// </summary>
 ///
 /// <param name="value"></param>
 /// <param name="result">The StringBuffer to write to.</param>
 public static void toEscapedString(ByteBuffer value_ren, StringBuilder result)
 {
     bool gotNonDot = false;
     for (int i = value_ren.position(); i < value_ren.limit(); ++i) {
         if (value_ren.get(i) != 0x2e) {
             gotNonDot = true;
             break;
         }
     }
     if (!gotNonDot) {
         // Special case for component of zero or more periods.  Add 3 periods.
         result.append("...");
         for (int i_0 = value_ren.position(); i_0 < value_ren.limit(); ++i_0)
             result.append('.');
     } else {
         for (int i_1 = value_ren.position(); i_1 < value_ren.limit(); ++i_1) {
             int x = ((int) value_ren.get(i_1) & 0xff);
             // Check for 0-9, A-Z, a-z, (+), (-), (.), (_)
             if (x >= 0x30 && x <= 0x39 || x >= 0x41 && x <= 0x5a
                     || x >= 0x61 && x <= 0x7a || x == 0x2b || x == 0x2d
                     || x == 0x2e || x == 0x5f)
                 result.append((char) x);
             else {
                 result.append('%');
                 if (x < 16)
                     result.append('0');
                 result.append(ILOG.J2CsMapping.Util.IlNumber.ToString(x,16).ToUpper());
             }
         }
     }
 }
示例#41
0
 /// <summary>
 /// Write this component value to result, escaping characters according to
 /// the NDN URI Scheme. This also adds "..." to a value with zero or more ".".
 /// This adds a type code prefix as needed, such as "sha256digest=".
 /// </summary>
 ///
 /// <param name="result">The StringBuffer to write to.</param>
 public void toEscapedString(StringBuilder result)
 {
     if (type_ == net.named_data.jndn.Name.Component.ComponentType.IMPLICIT_SHA256_DIGEST) {
         result.append("sha256digest=");
         net.named_data.jndn.util.Blob.toHex(value_.buf(), result);
     } else
         net.named_data.jndn.Name.toEscapedString(value_.buf(), result);
 }
示例#42
-5
    // A1: going in reverse order, detect each word to append to the new string
    public String reverseWords(String a)
    {
        if (a == null || a.length() == 0) return a;

        StringBuilder b = new StringBuilder();

        int end = -1;
        int start = -1;
        int i = a.length()-1;

        while (i >= 0)
        {
            while (i >= 0 && a.charAt(i) == ' ') i--;
            end = i;

            while (i >= 0 && a.charAt(i) != ' ') i--;
            start = i;

            if (start < end) {
                if (b.length() > 0) b.append(' ');
                b.append(a.substring(start+1, end+1));
            }
        }

        return b.toString();
    }