public ResultThread(Fetch init_fetch, FetchHandler init_handler, HTTPRawReader init_reader) { fetch = init_fetch; handler = init_handler; reader = init_reader; exception = null; }
private void doSaveResponsesSetup() { Debug.Assert(reader != null); if ((Environment.GetEnvironmentVariable( "GOLDEN_RETRIEVER_SAVE_PREFIX") != null) && (Environment.GetEnvironmentVariable( "GOLDEN_RETRIEVER_SAVE_CATALOG") != null)) { reader = new SaveReader(reader, url); } }
public SaveReader(HTTPRawReader init_next, string url) { Debug.Assert(init_next != null); Debug.Assert(url != null); next = init_next; string prefix = Environment.GetEnvironmentVariable( "GOLDEN_RETRIEVER_SAVE_PREFIX"); ++save_file_counter; string save_file_name = prefix + DateTime.UtcNow.ToString("_yyyy_MM_dd_HH_mm_ss") + string.Format("_{0:4}.txt", save_file_counter); try { save_stream = new FileStream(save_file_name, FileMode.Create); save_writer = new StreamWriter(save_stream); } catch (Exception e1) { Console.Error.Write( "Warning: Failed trying to open save file {0}: {1}.\n", save_file_name, e1.Message); save_stream = null; } try { FileStream catalog_stream = new FileStream( Environment.GetEnvironmentVariable( "GOLDEN_RETRIEVER_SAVE_CATALOG"), FileMode.Create); new StreamWriter(catalog_stream).Write( " [\"{0}\", \"{1}\"],\n", url, save_file_name); catalog_stream.Close(); } catch (Exception e1) { Console.Error.Write( "Warning: Failed trying to open save catalog file " + "{0}: {1}.\n", Environment.GetEnvironmentVariable( "GOLDEN_RETRIEVER_SAVE_CATALOG"), e1.Message); } }
private void doPreHeader(string method) { if (!(FetchEnable.fetches_are_enabled())) { Console.Error.Write( "ERROR: An attempt was made to hit an external API " + "(URL {0}) while such hits were disabled because {1}.\n", url, FetchEnable.fetch_disable_reason()); Environment.Exit(1); } URLInfo url_info = parseURL(url); if (url_info.is_script) { int token_count = 1; int follow = url_info.host_start; while (follow < url_info.path_start + url_info.path_length) { if (((url[follow] == ' ') || (url[follow] == '@')) && ((follow == url.Length) || (url[follow + 1] != ' ') || (url[follow + 1] != '@'))) { ++token_count; } ++follow; } string file_name = ""; string arguments = ""; int start = url_info.host_start; int arg_num = 1; int follow_buffer = start; while (follow_buffer < (url_info.path_start + url_info.path_length)) { if ((url[follow_buffer] == ' ') || (url[follow_buffer] == '@')) { string this_arg = url.Substring(start, follow_buffer - start); if (arg_num == 1) { file_name = this_arg; } else if (arg_num == 2) { arguments = this_arg; } else { arguments += " "; arguments += this_arg; } if ((follow_buffer == url.Length) || (url[follow_buffer + 1] != ' ') || (url[follow_buffer + 1] != '@')) { start = follow_buffer + 1; ++arg_num; } } ++follow_buffer; } Debug.Assert(arg_num == token_count); string final_arg = url.Substring(start, follow_buffer - start); if (arg_num == 1) { file_name = final_arg; } else if (arg_num == 2) { arguments = final_arg; } else { arguments += " "; arguments += final_arg; } Process child = new Process { StartInfo = new ProcessStartInfo { FileName = file_name, Arguments = arguments, UseShellExecute = false, RedirectStandardInput = true, RedirectStandardOutput = true, CreateNoWindow = true } }; child.Start(); reader = new ScriptReader(child); writer = new FetchStreamWriter(child.StandardInput.BaseStream, reader); doSaveResponsesSetup(); return; } if (Environment.GetEnvironmentVariable( "GOLDEN_RETRIEVER_SHOW_REAL_URL_HITS") != null) { Console.Error.Write("Golden Retriever hitting `{0}'.\n", url); } string final_hostname = url.Substring(url_info.host_start, url_info.host_length); string hostname = (use_proxy ? proxy_server : final_hostname); int port_to_use = (use_proxy ? proxy_port : url_info.port); NetworkStream stream = new TcpClient(hostname, port_to_use).GetStream(); if (url_info.is_ssl) { if (use_proxy) { HTTPStreamReader proxy_reader = new HTTPStreamReader(stream, 1); FetchStreamWriter proxy_writer = new FetchStreamWriter(stream, proxy_reader); proxy_writer.write("CONNECT "); proxy_writer.write(final_hostname); proxy_writer.write(":" + url_info.port); proxy_writer.write(" HTTP/1.1\r\n"); proxy_writer.write(user_agent_key); proxy_writer.write(": "); proxy_writer.write( (proxy_user_agent != null) ? proxy_user_agent : user_agent); proxy_writer.write("\r\n"); int param_count = proxy_parameters.Count; for (int param_num = 0; param_num < param_count; ++param_num) { proxy_writer.write(proxy_parameters[param_num].key); proxy_writer.write(": "); proxy_writer.write(proxy_parameters[param_num].value); proxy_writer.write("\r\n"); } proxy_writer.write("\r\n"); string first_line = proxy_reader.readLine(); string[] first_components = first_line.Split(new Char[] { ' ' }); if (first_components.Length < 3) { throw new Exception( "Bad first line from proxy server: `" + first_line + "'."); } if (!(first_components[1].Equals("200"))) { string message = "Proxy returned status code "; message += first_components[1]; message += ":"; for (int num = 2; num < first_components.Length; ++num) { message += " " + first_components[num]; } message += "."; throw new Exception(message); } while (true) { string next_line = proxy_reader.readLine(); if (next_line.Equals("") || next_line.Equals("\r")) { break; } } } SslStream ssl_stream = new SslStream(stream); ssl_stream.AuthenticateAsClient(final_hostname); reader = new HTTPStreamReader(ssl_stream); writer = new FetchStreamWriter(ssl_stream, reader); } else { reader = new HTTPStreamReader(stream); writer = new FetchStreamWriter(stream, reader); } doSaveResponsesSetup(); }
public FetchStreamWriter(Stream init_stream, HTTPRawReader init_reader) { stream = init_stream; reader = init_reader; Debug.Assert(reader != null); }
public void handleParametersAndBody(HTTPHandler handler, HTTPRawReader reader) { while (true) { string this_line = reader.readLine(); if ((this_line == null) || (this_line.Length == 0)) { break; } handle_header_line(this_line, handler); } handler.handleHeaderDone(); byte[] buffer = new byte[256]; if (chunked) { while (true) { long chunk_byte_count = 0; while (true) { int num = reader.read_bytes(buffer, 1); if (num < 1) { throw new Exception("Unexpected end of chunk header."); } if (buffer[0] == '\r') { break; } if (buffer[0] == ' ') { continue; } long digit; if ((buffer[0] >= '0') && (buffer[0] <= '9')) { digit = buffer[0] - '0'; } else if ((buffer[0] >= 'a') && (buffer[0] <= 'f')) { digit = (buffer[0] - 'a') + 0xa; } else if ((buffer[0] >= 'A') && (buffer[0] <= 'F')) { digit = (buffer[0] - 'A') + 0xa; } else { throw new Exception("Bad digit in chunk header."); } if (chunk_byte_count >= (Int64.MaxValue - digit) / 16) { throw new Exception("Overflow in chunk header."); } chunk_byte_count = (chunk_byte_count * 16) + digit; } if (chunk_byte_count == 0) { break; } { int num = reader.read_bytes(buffer, 1); if (num < 1) { throw new Exception("Unexpected end of chunk header."); } if (buffer[0] != '\n') { throw new Exception("Bad end-of-line in chunk header."); } } while (chunk_byte_count > 0) { int num = reader.read_bytes(buffer, ((chunk_byte_count > 256) ? 256 : (int)chunk_byte_count)); if (num < 1) { break; } handler.handleContent(num, buffer); chunk_byte_count -= num; } { int num = reader.read_bytes(buffer, 1); if (num < 1) { throw new Exception("Unexpected end of chunk."); } if (buffer[0] != '\r') { throw new Exception("Bad end-of-line in chunk."); } } { int num = reader.read_bytes(buffer, 1); if (num < 1) { throw new Exception("Unexpected end of chunk."); } if (buffer[0] != '\n') { throw new Exception("Bad end-of-line in chunk."); } } } /* @@@ */ /* @@@ -- Read the trailer. */ /* @@@ */ } else if (have_content_length) { while (content_length > 0) { int num = reader.read_bytes(buffer, ((content_length > 256) ? 256 : (int)content_length)); if (num < 1) { break; } Debug.Assert(num <= content_length); handler.handleContent(num, buffer); content_length -= num; } } else { while (true) { int num = reader.read_bytes(buffer, 256); if (num < 1) { break; } handler.handleContent(num, buffer); } } handler.handleContentEnd(); }