public ResultThread(Fetch init_fetch, FetchHandler init_handler,
HTTPRawReader init_reader)
{
fetch = init_fetch;
handler = init_handler;
reader = init_reader;
exception = null;
}
private void doSaveResponsesSetup()
{
Debug.Assert(reader != null);
if ((Environment.GetEnvironmentVariable(
"GOLDEN_RETRIEVER_SAVE_PREFIX") != null) &&
(Environment.GetEnvironmentVariable(
"GOLDEN_RETRIEVER_SAVE_CATALOG") != null))
{
reader = new SaveReader(reader, url);
}
}
public SaveReader(HTTPRawReader init_next, string url)
{
Debug.Assert(init_next != null);
Debug.Assert(url != null);
next = init_next;
string prefix = Environment.GetEnvironmentVariable(
"GOLDEN_RETRIEVER_SAVE_PREFIX");
++save_file_counter;
string save_file_name =
prefix + DateTime.UtcNow.ToString("_yyyy_MM_dd_HH_mm_ss") +
string.Format("_{0:4}.txt", save_file_counter);
try
{
save_stream = new FileStream(save_file_name, FileMode.Create);
save_writer = new StreamWriter(save_stream);
}
catch (Exception e1)
{
Console.Error.Write(
"Warning: Failed trying to open save file {0}: {1}.\n",
save_file_name, e1.Message);
save_stream = null;
}
try
{
FileStream catalog_stream = new FileStream(
Environment.GetEnvironmentVariable(
"GOLDEN_RETRIEVER_SAVE_CATALOG"),
FileMode.Create);
new StreamWriter(catalog_stream).Write(
" [\"{0}\", \"{1}\"],\n", url, save_file_name);
catalog_stream.Close();
}
catch (Exception e1)
{
Console.Error.Write(
"Warning: Failed trying to open save catalog file " +
"{0}: {1}.\n",
Environment.GetEnvironmentVariable(
"GOLDEN_RETRIEVER_SAVE_CATALOG"), e1.Message);
}
}
private void doPreHeader(string method)
{
if (!(FetchEnable.fetches_are_enabled()))
{
Console.Error.Write(
"ERROR: An attempt was made to hit an external API " +
"(URL {0}) while such hits were disabled because {1}.\n",
url, FetchEnable.fetch_disable_reason());
Environment.Exit(1);
}
URLInfo url_info = parseURL(url);
if (url_info.is_script)
{
int token_count = 1;
int follow = url_info.host_start;
while (follow < url_info.path_start + url_info.path_length)
{
if (((url[follow] == ' ') || (url[follow] == '@')) &&
((follow == url.Length) || (url[follow + 1] != ' ') ||
(url[follow + 1] != '@')))
{
++token_count;
}
++follow;
}
string file_name = "";
string arguments = "";
int start = url_info.host_start;
int arg_num = 1;
int follow_buffer = start;
while (follow_buffer <
(url_info.path_start + url_info.path_length))
{
if ((url[follow_buffer] == ' ') || (url[follow_buffer] == '@'))
{
string this_arg =
url.Substring(start, follow_buffer - start);
if (arg_num == 1)
{
file_name = this_arg;
}
else if (arg_num == 2)
{
arguments = this_arg;
}
else
{
arguments += " ";
arguments += this_arg;
}
if ((follow_buffer == url.Length) ||
(url[follow_buffer + 1] != ' ') ||
(url[follow_buffer + 1] != '@'))
{
start = follow_buffer + 1;
++arg_num;
}
}
++follow_buffer;
}
Debug.Assert(arg_num == token_count);
string final_arg = url.Substring(start, follow_buffer - start);
if (arg_num == 1)
{
file_name = final_arg;
}
else if (arg_num == 2)
{
arguments = final_arg;
}
else
{
arguments += " ";
arguments += final_arg;
}
Process child = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = file_name,
Arguments = arguments,
UseShellExecute = false,
RedirectStandardInput = true,
RedirectStandardOutput = true,
CreateNoWindow = true
}
};
child.Start();
reader = new ScriptReader(child);
writer = new FetchStreamWriter(child.StandardInput.BaseStream,
reader);
doSaveResponsesSetup();
return;
}
if (Environment.GetEnvironmentVariable(
"GOLDEN_RETRIEVER_SHOW_REAL_URL_HITS") != null)
{
Console.Error.Write("Golden Retriever hitting `{0}'.\n", url);
}
string final_hostname =
url.Substring(url_info.host_start, url_info.host_length);
string hostname = (use_proxy ? proxy_server : final_hostname);
int port_to_use = (use_proxy ? proxy_port : url_info.port);
NetworkStream stream = new TcpClient(hostname, port_to_use).GetStream();
if (url_info.is_ssl)
{
if (use_proxy)
{
HTTPStreamReader proxy_reader = new HTTPStreamReader(stream, 1);
FetchStreamWriter proxy_writer =
new FetchStreamWriter(stream, proxy_reader);
proxy_writer.write("CONNECT ");
proxy_writer.write(final_hostname);
proxy_writer.write(":" + url_info.port);
proxy_writer.write(" HTTP/1.1\r\n");
proxy_writer.write(user_agent_key);
proxy_writer.write(": ");
proxy_writer.write(
(proxy_user_agent != null) ? proxy_user_agent :
user_agent);
proxy_writer.write("\r\n");
int param_count = proxy_parameters.Count;
for (int param_num = 0; param_num < param_count; ++param_num)
{
proxy_writer.write(proxy_parameters[param_num].key);
proxy_writer.write(": ");
proxy_writer.write(proxy_parameters[param_num].value);
proxy_writer.write("\r\n");
}
proxy_writer.write("\r\n");
string first_line = proxy_reader.readLine();
string[] first_components = first_line.Split(new Char[] { ' ' });
if (first_components.Length < 3)
{
throw new Exception(
"Bad first line from proxy server: `" + first_line +
"'.");
}
if (!(first_components[1].Equals("200")))
{
string message = "Proxy returned status code ";
message += first_components[1];
message += ":";
for (int num = 2; num < first_components.Length; ++num)
{
message += " " + first_components[num];
}
message += ".";
throw new Exception(message);
}
while (true)
{
string next_line = proxy_reader.readLine();
if (next_line.Equals("") || next_line.Equals("\r"))
{
break;
}
}
}
SslStream ssl_stream = new SslStream(stream);
ssl_stream.AuthenticateAsClient(final_hostname);
reader = new HTTPStreamReader(ssl_stream);
writer =
new FetchStreamWriter(ssl_stream, reader);
}
else
{
reader = new HTTPStreamReader(stream);
writer = new FetchStreamWriter(stream, reader);
}
doSaveResponsesSetup();
}
public FetchStreamWriter(Stream init_stream, HTTPRawReader init_reader)
{
stream = init_stream;
reader = init_reader;
Debug.Assert(reader != null);
}
public void handleParametersAndBody(HTTPHandler handler,
HTTPRawReader reader)
{
while (true)
{
string this_line = reader.readLine();
if ((this_line == null) || (this_line.Length == 0))
{
break;
}
handle_header_line(this_line, handler);
}
handler.handleHeaderDone();
byte[] buffer = new byte[256];
if (chunked)
{
while (true)
{
long chunk_byte_count = 0;
while (true)
{
int num = reader.read_bytes(buffer, 1);
if (num < 1)
{
throw new Exception("Unexpected end of chunk header.");
}
if (buffer[0] == '\r')
{
break;
}
if (buffer[0] == ' ')
{
continue;
}
long digit;
if ((buffer[0] >= '0') && (buffer[0] <= '9'))
{
digit = buffer[0] - '0';
}
else if ((buffer[0] >= 'a') && (buffer[0] <= 'f'))
{
digit = (buffer[0] - 'a') + 0xa;
}
else if ((buffer[0] >= 'A') && (buffer[0] <= 'F'))
{
digit = (buffer[0] - 'A') + 0xa;
}
else
{
throw new Exception("Bad digit in chunk header.");
}
if (chunk_byte_count >= (Int64.MaxValue - digit) / 16)
{
throw new Exception("Overflow in chunk header.");
}
chunk_byte_count = (chunk_byte_count * 16) + digit;
}
if (chunk_byte_count == 0)
{
break;
}
{
int num = reader.read_bytes(buffer, 1);
if (num < 1)
{
throw new Exception("Unexpected end of chunk header.");
}
if (buffer[0] != '\n')
{
throw new Exception("Bad end-of-line in chunk header.");
}
}
while (chunk_byte_count > 0)
{
int num = reader.read_bytes(buffer,
((chunk_byte_count > 256) ? 256 :
(int)chunk_byte_count));
if (num < 1)
{
break;
}
handler.handleContent(num, buffer);
chunk_byte_count -= num;
}
{
int num = reader.read_bytes(buffer, 1);
if (num < 1)
{
throw new Exception("Unexpected end of chunk.");
}
if (buffer[0] != '\r')
{
throw new Exception("Bad end-of-line in chunk.");
}
}
{
int num = reader.read_bytes(buffer, 1);
if (num < 1)
{
throw new Exception("Unexpected end of chunk.");
}
if (buffer[0] != '\n')
{
throw new Exception("Bad end-of-line in chunk.");
}
}
}
/* @@@ */
/* @@@ -- Read the trailer. */
/* @@@ */
}
else if (have_content_length)
{
while (content_length > 0)
{
int num = reader.read_bytes(buffer,
((content_length > 256) ? 256 : (int)content_length));
if (num < 1)
{
break;
}
Debug.Assert(num <= content_length);
handler.handleContent(num, buffer);
content_length -= num;
}
}
else
{
while (true)
{
int num = reader.read_bytes(buffer, 256);
if (num < 1)
{
break;
}
handler.handleContent(num, buffer);
}
}
handler.handleContentEnd();
}