Exemplo n.º 1
0
 public ResultThread(Fetch init_fetch, FetchHandler init_handler,
                     HTTPRawReader init_reader)
 {
     fetch     = init_fetch;
     handler   = init_handler;
     reader    = init_reader;
     exception = null;
 }
Exemplo n.º 2
0
    private void doSaveResponsesSetup()
    {
        Debug.Assert(reader != null);

        if ((Environment.GetEnvironmentVariable(
                 "GOLDEN_RETRIEVER_SAVE_PREFIX") != null) &&
            (Environment.GetEnvironmentVariable(
                 "GOLDEN_RETRIEVER_SAVE_CATALOG") != null))
        {
            reader = new SaveReader(reader, url);
        }
    }
Exemplo n.º 3
0
        public SaveReader(HTTPRawReader init_next, string url)
        {
            Debug.Assert(init_next != null);
            Debug.Assert(url != null);

            next = init_next;

            string prefix = Environment.GetEnvironmentVariable(
                "GOLDEN_RETRIEVER_SAVE_PREFIX");

            ++save_file_counter;
            string save_file_name =
                prefix + DateTime.UtcNow.ToString("_yyyy_MM_dd_HH_mm_ss") +
                string.Format("_{0:4}.txt", save_file_counter);

            try
            {
                save_stream = new FileStream(save_file_name, FileMode.Create);
                save_writer = new StreamWriter(save_stream);
            }
            catch (Exception e1)
            {
                Console.Error.Write(
                    "Warning: Failed trying to open save file {0}: {1}.\n",
                    save_file_name, e1.Message);
                save_stream = null;
            }
            try
            {
                FileStream catalog_stream = new FileStream(
                    Environment.GetEnvironmentVariable(
                        "GOLDEN_RETRIEVER_SAVE_CATALOG"),
                    FileMode.Create);
                new StreamWriter(catalog_stream).Write(
                    "  [\"{0}\", \"{1}\"],\n", url, save_file_name);
                catalog_stream.Close();
            }
            catch (Exception e1)
            {
                Console.Error.Write(
                    "Warning: Failed trying to open save catalog file " +
                    "{0}: {1}.\n",
                    Environment.GetEnvironmentVariable(
                        "GOLDEN_RETRIEVER_SAVE_CATALOG"), e1.Message);
            }
        }
Exemplo n.º 4
0
    private void doPreHeader(string method)
    {
        if (!(FetchEnable.fetches_are_enabled()))
        {
            Console.Error.Write(
                "ERROR: An attempt was made to hit an external API " +
                "(URL {0}) while such hits were disabled because {1}.\n",
                url, FetchEnable.fetch_disable_reason());
            Environment.Exit(1);
        }

        URLInfo url_info = parseURL(url);

        if (url_info.is_script)
        {
            int token_count = 1;
            int follow      = url_info.host_start;
            while (follow < url_info.path_start + url_info.path_length)
            {
                if (((url[follow] == ' ') || (url[follow] == '@')) &&
                    ((follow == url.Length) || (url[follow + 1] != ' ') ||
                     (url[follow + 1] != '@')))
                {
                    ++token_count;
                }
                ++follow;
            }
            string file_name     = "";
            string arguments     = "";
            int    start         = url_info.host_start;
            int    arg_num       = 1;
            int    follow_buffer = start;
            while (follow_buffer <
                   (url_info.path_start + url_info.path_length))
            {
                if ((url[follow_buffer] == ' ') || (url[follow_buffer] == '@'))
                {
                    string this_arg =
                        url.Substring(start, follow_buffer - start);
                    if (arg_num == 1)
                    {
                        file_name = this_arg;
                    }
                    else if (arg_num == 2)
                    {
                        arguments = this_arg;
                    }
                    else
                    {
                        arguments += " ";
                        arguments += this_arg;
                    }
                    if ((follow_buffer == url.Length) ||
                        (url[follow_buffer + 1] != ' ') ||
                        (url[follow_buffer + 1] != '@'))
                    {
                        start = follow_buffer + 1;
                        ++arg_num;
                    }
                }
                ++follow_buffer;
            }
            Debug.Assert(arg_num == token_count);
            string final_arg = url.Substring(start, follow_buffer - start);
            if (arg_num == 1)
            {
                file_name = final_arg;
            }
            else if (arg_num == 2)
            {
                arguments = final_arg;
            }
            else
            {
                arguments += " ";
                arguments += final_arg;
            }

            Process child = new Process
            {
                StartInfo = new ProcessStartInfo
                {
                    FileName               = file_name,
                    Arguments              = arguments,
                    UseShellExecute        = false,
                    RedirectStandardInput  = true,
                    RedirectStandardOutput = true,
                    CreateNoWindow         = true
                }
            };
            child.Start();
            reader = new ScriptReader(child);

            writer = new FetchStreamWriter(child.StandardInput.BaseStream,
                                           reader);

            doSaveResponsesSetup();

            return;
        }

        if (Environment.GetEnvironmentVariable(
                "GOLDEN_RETRIEVER_SHOW_REAL_URL_HITS") != null)
        {
            Console.Error.Write("Golden Retriever hitting `{0}'.\n", url);
        }

        string final_hostname =
            url.Substring(url_info.host_start, url_info.host_length);
        string hostname    = (use_proxy ? proxy_server : final_hostname);
        int    port_to_use = (use_proxy ? proxy_port : url_info.port);

        NetworkStream stream = new TcpClient(hostname, port_to_use).GetStream();

        if (url_info.is_ssl)
        {
            if (use_proxy)
            {
                HTTPStreamReader  proxy_reader = new HTTPStreamReader(stream, 1);
                FetchStreamWriter proxy_writer =
                    new FetchStreamWriter(stream, proxy_reader);
                proxy_writer.write("CONNECT ");
                proxy_writer.write(final_hostname);
                proxy_writer.write(":" + url_info.port);
                proxy_writer.write(" HTTP/1.1\r\n");
                proxy_writer.write(user_agent_key);
                proxy_writer.write(": ");
                proxy_writer.write(
                    (proxy_user_agent != null) ? proxy_user_agent :
                    user_agent);
                proxy_writer.write("\r\n");
                int param_count = proxy_parameters.Count;
                for (int param_num = 0; param_num < param_count; ++param_num)
                {
                    proxy_writer.write(proxy_parameters[param_num].key);
                    proxy_writer.write(": ");
                    proxy_writer.write(proxy_parameters[param_num].value);
                    proxy_writer.write("\r\n");
                }
                proxy_writer.write("\r\n");

                string   first_line       = proxy_reader.readLine();
                string[] first_components = first_line.Split(new Char[] { ' ' });
                if (first_components.Length < 3)
                {
                    throw new Exception(
                              "Bad first line from proxy server: `" + first_line +
                              "'.");
                }
                if (!(first_components[1].Equals("200")))
                {
                    string message = "Proxy returned status code ";
                    message += first_components[1];
                    message += ":";
                    for (int num = 2; num < first_components.Length; ++num)
                    {
                        message += " " + first_components[num];
                    }
                    message += ".";
                    throw new Exception(message);
                }

                while (true)
                {
                    string next_line = proxy_reader.readLine();
                    if (next_line.Equals("") || next_line.Equals("\r"))
                    {
                        break;
                    }
                }
            }
            SslStream ssl_stream = new SslStream(stream);
            ssl_stream.AuthenticateAsClient(final_hostname);
            reader = new HTTPStreamReader(ssl_stream);
            writer =
                new FetchStreamWriter(ssl_stream, reader);
        }
        else
        {
            reader = new HTTPStreamReader(stream);
            writer = new FetchStreamWriter(stream, reader);
        }

        doSaveResponsesSetup();
    }
Exemplo n.º 5
0
 public FetchStreamWriter(Stream init_stream, HTTPRawReader init_reader)
 {
     stream = init_stream;
     reader = init_reader;
     Debug.Assert(reader != null);
 }
    public void handleParametersAndBody(HTTPHandler handler,
                                        HTTPRawReader reader)
    {
        while (true)
        {
            string this_line = reader.readLine();
            if ((this_line == null) || (this_line.Length == 0))
            {
                break;
            }
            handle_header_line(this_line, handler);
        }

        handler.handleHeaderDone();

        byte[] buffer = new byte[256];

        if (chunked)
        {
            while (true)
            {
                long chunk_byte_count = 0;
                while (true)
                {
                    int num = reader.read_bytes(buffer, 1);
                    if (num < 1)
                    {
                        throw new Exception("Unexpected end of chunk header.");
                    }
                    if (buffer[0] == '\r')
                    {
                        break;
                    }
                    if (buffer[0] == ' ')
                    {
                        continue;
                    }
                    long digit;
                    if ((buffer[0] >= '0') && (buffer[0] <= '9'))
                    {
                        digit = buffer[0] - '0';
                    }
                    else if ((buffer[0] >= 'a') && (buffer[0] <= 'f'))
                    {
                        digit = (buffer[0] - 'a') + 0xa;
                    }
                    else if ((buffer[0] >= 'A') && (buffer[0] <= 'F'))
                    {
                        digit = (buffer[0] - 'A') + 0xa;
                    }
                    else
                    {
                        throw new Exception("Bad digit in chunk header.");
                    }
                    if (chunk_byte_count >= (Int64.MaxValue - digit) / 16)
                    {
                        throw new Exception("Overflow in chunk header.");
                    }
                    chunk_byte_count = (chunk_byte_count * 16) + digit;
                }

                if (chunk_byte_count == 0)
                {
                    break;
                }

                {
                    int num = reader.read_bytes(buffer, 1);
                    if (num < 1)
                    {
                        throw new Exception("Unexpected end of chunk header.");
                    }
                    if (buffer[0] != '\n')
                    {
                        throw new Exception("Bad end-of-line in chunk header.");
                    }
                }

                while (chunk_byte_count > 0)
                {
                    int num = reader.read_bytes(buffer,
                                                ((chunk_byte_count > 256) ? 256 :
                                                 (int)chunk_byte_count));
                    if (num < 1)
                    {
                        break;
                    }
                    handler.handleContent(num, buffer);
                    chunk_byte_count -= num;
                }

                {
                    int num = reader.read_bytes(buffer, 1);
                    if (num < 1)
                    {
                        throw new Exception("Unexpected end of chunk.");
                    }
                    if (buffer[0] != '\r')
                    {
                        throw new Exception("Bad end-of-line in chunk.");
                    }
                }

                {
                    int num = reader.read_bytes(buffer, 1);
                    if (num < 1)
                    {
                        throw new Exception("Unexpected end of chunk.");
                    }
                    if (buffer[0] != '\n')
                    {
                        throw new Exception("Bad end-of-line in chunk.");
                    }
                }
            }

            /* @@@ */
            /* @@@ -- Read the trailer. */
            /* @@@ */
        }
        else if (have_content_length)
        {
            while (content_length > 0)
            {
                int num = reader.read_bytes(buffer,
                                            ((content_length > 256) ? 256 : (int)content_length));
                if (num < 1)
                {
                    break;
                }
                Debug.Assert(num <= content_length);
                handler.handleContent(num, buffer);
                content_length -= num;
            }
        }
        else
        {
            while (true)
            {
                int num = reader.read_bytes(buffer, 256);
                if (num < 1)
                {
                    break;
                }
                handler.handleContent(num, buffer);
            }
        }
        handler.handleContentEnd();
    }