/// <summary>
/// Adds the request to the global queue and wakes up the dispatcher.
/// </summary>
/// <param name="requestHandler">The request handler to queue.</param>
public void AddRequest(RemoteRequestHandler requestHandler)
{
requestHandler = (RemoteRequestHandler) AddRequestGlobalQueue(requestHandler);
// Notify that a new request has been added. The Dispatcher will wake up if it was waiting.
_requestEvent.Set();
}
/// <summary>
/// Adds all the requests to the package.
/// </summary>
/// <param name="requests"></param>
/// <returns></returns>
public LinkedList<RCRequest> Pack(RemoteRequestHandler requestHandler, LinkedList<RCRequest> requests, ref long quota)
{
LinkedList<RCRequest> addedObjects = new LinkedList<RCRequest>();
// add files that were completed to the package
foreach (RCRequest request in requests)
{
// check watermark
//if (quota > requestHandler.DEFAULT_LOW_WATERMARK)
//{
// add to the package
if (Pack(requestHandler, request, ref quota))
{
//requestHandler.LogDebug("packed: " + request.Uri + " " + request.FileSize + " bytes - " + quota + " left");
addedObjects.AddLast(request);
}
//}
}
return addedObjects;
}
/// <summary>
/// Adds a RCRequest to the package given the quota limitation.
/// Only called by the remote proxy.
/// XXX: quota is currently simple algorithm.
/// XXX: should be changed to check for compressed size rather than actual size.
/// </summary>
/// <param name="requestHandler">Calling handler for this method.</param>
/// <param name="rcRequest">RCRequest to add.</param>
/// <param name="quota">Quota limit.</param>
/// <returns>Error message.</returns>
public bool Pack(RemoteRequestHandler requestHandler, RCRequest rcRequest, ref long quota)
{
if (rcRequest.Uri.Contains(' '))
{
requestHandler.LogDebug("object contains spaces: " + rcRequest.Uri);
return false;
}
if (_rcRequests.Contains(rcRequest))
{
requestHandler.LogDebug("object exists in package: " + rcRequest.Uri);
return false;
}
rcRequest.FileSize = Util.GetFileSize(rcRequest.CacheFileName);
if (rcRequest.FileSize <= 0)
{
//requestHandler.LogDebug("object has no content: " + rcRequest.Uri);
return false;
}
// quota check
if ((quota - rcRequest.FileSize) < 0)
{
requestHandler.LogDebug("object doesn't fit in quota: " + rcRequest.Uri);
return false;
}
_rcRequests.AddLast(rcRequest);
quota -= rcRequest.FileSize;
//requestHandler.LogDebug("packed: " + requestHandler.RequestUri + " " + rcRequest.FileSize + " bytes - " + quota + " left");
return true;
}
/// <summary>
/// Counts embedded objects and links on a page.
/// Currently only takes in simple log format of "google.txt".
/// Unused.
/// </summary>
public static void CountEmbeddedObjects()
{
Console.WindowWidth = Console.LargestWindowWidth;
Console.WindowHeight = Console.LargestWindowHeight;
Console.SetWindowPosition(0, 0);
// load Configuration Settings
Program.SaveConfigs();
// start the local proxy
//StartLocalProxy();
// start the remote proxy
RCRemoteProxy remoteProxy = Program.StartRemoteProxy();
// parse the input file line by line
// create reader & open file
TextReader tr = new StreamReader("google.txt");
uint linesParsed = 0;
uint requestsMade = 0;
string line = tr.ReadLine();
string[] lineTokens;
while (line != null)
{
linesParsed++;
line = line.ToLower();
lineTokens = line.Split('\t');
// maximum number of tokens is 100
if (lineTokens.Length >= 100 || lineTokens.Length < 19)
{
_logger.Warn("too many tokens to fit in array");
// read the next line
line = tr.ReadLine();
continue;
}
// make sure that its actually a search query
string urlRequest = lineTokens[18];
if (urlRequest.StartsWith("http://www.google.com/search?"))
{
if (urlRequest.Length > 200)
{
line = tr.ReadLine();
continue;
}
_logger.Info(urlRequest);
// process search query one at a time
RemoteRequestHandler requestHandler = new RemoteRequestHandler(remoteProxy, null);
if (HttpUtils.IsValidUri(urlRequest))
{
requestHandler.RCRequest = new RCRequest(requestHandler, (HttpWebRequest) WebRequest.Create(urlRequest.Trim()));
requestHandler.RCRequest.SetProxyAndTimeout(remoteProxy.GatewayProxy, RemoteRequestHandler.WEB_REQUEST_DEFAULT_TIMEOUT);
//requestHandler.PrefetchAnalysis("high", 1);
Thread.Sleep(1500);
}
}
requestsMade++;
// stop after 1000 entries
if (requestsMade == 1000)
{
//SaveBenchmarkValues("linksOnResultsPage.out", RPRequestHandler.linksOnResultsPage);
//SaveBenchmarkValues("imagesOnResultsPage.out", RPRequestHandler.imagesOnResultsPage);
//SaveBenchmarkValues("imagesOnTargetPage.out", RPRequestHandler.imagesOnTargetPage);
}
// read the next line
line = tr.ReadLine();
}
tr.Close();
// keep the console open
Console.ReadLine();
}
/// <summary>
/// Starts the listener for connections from local proxy.
/// The remote proxy could potentially serve multiple local proxies.
/// </summary>
public override void StartListener()
{
WriteDebug("Started Listener on " +
_listenAddress + ":" + _listenPort);
try
{
// create a listener for the proxy port
TcpListener sockServer = new TcpListener(_listenAddress, _listenPort);
sockServer.Start();
// loop and listen for the next connection request
while (true)
{
// accept connections on the proxy port (blocks)
Socket socket = sockServer.AcceptSocket();
// handle the accepted connection in a separate thread
RemoteRequestHandler requestHandler = new RemoteRequestHandler(this, socket);
Thread proxyThread = new Thread(new ThreadStart(requestHandler.Go));
proxyThread.Start();
}
}
catch (SocketException ex)
{
WriteDebug("SocketException in StartRemoteListener, errorcode: " + ex.NativeErrorCode);
}
catch (Exception e)
{
WriteDebug("Exception in StartRemoteListener: " + e.StackTrace + " " + e.Message);
}
}
/// <summary>
/// Counts embedded objects and links on a page.
/// Currently only takes in simple log format of "google.txt".
/// Unused.
/// </summary>
public static void CountEmbeddedObjects()
{
Console.WindowWidth = Console.LargestWindowWidth;
Console.WindowHeight = Console.LargestWindowHeight;
Console.SetWindowPosition(0, 0);
// fill extension map
Util.FillExtMap();
// load Configuration Settings
Program.LoadConfigFile();
// start the local proxy
//StartLocalProxy();
// start the remote proxy
RCRemoteProxy remoteProxy = Program.StartRemoteProxy();
// parse the input file line by line
// create reader & open file
TextReader tr = new StreamReader("google.txt");
uint linesParsed = 0;
uint requestsMade = 0;
string line = tr.ReadLine();
string[] lineTokens;
while (line != null)
{
linesParsed++;
line = line.ToLower();
lineTokens = line.Split('\t');
// maximum number of tokens is 100
if (lineTokens.Length >= 100 || lineTokens.Length < 19)
{
Console.WriteLine("Error, too many tokens to fit in array");
// read the next line
line = tr.ReadLine();
continue;
}
// make sure that its actually a search query
string urlRequest = lineTokens[18];
if (urlRequest.StartsWith("http://www.google.com/search?"))
{
if (urlRequest.Length > 200)
{
line = tr.ReadLine();
continue;
}
/* print to show that we're processing
if (DateTime.Now.Subtract(displayDotTimer).TotalMinutes > 0)
{
Console.Write(".");
displayDotTimer = DateTime.Now;
}*/
Console.WriteLine(urlRequest);
/*
// check if the query has already been processed before
if (IsCached(fileName))
{
// skip cached files
continue;
}
*/
// process search query one at a time
RemoteRequestHandler requestHandler = new RemoteRequestHandler(remoteProxy, null);
if (Util.IsValidUri(urlRequest))
{
requestHandler.RCRequest = new RCRequest(requestHandler, urlRequest);
requestHandler.RCRequest.SetProxyAndTimeout(remoteProxy.GatewayProxy, RemoteRequestHandler.WEB_REQUEST_DEFAULT_TIMEOUT);
//requestHandler.PrefetchAnalysis("high", 1);
Thread.Sleep(1500);
}
}
requestsMade++;
// stop after 1000 entries
if (requestsMade == 1000)
{
//SaveBenchmarkValues("linksOnResultsPage.out", RPRequestHandler.linksOnResultsPage);
//SaveBenchmarkValues("imagesOnResultsPage.out", RPRequestHandler.imagesOnResultsPage);
//SaveBenchmarkValues("imagesOnTargetPage.out", RPRequestHandler.imagesOnTargetPage);
}
// read the next line
line = tr.ReadLine();
}
tr.Close();
// keep the console open
Console.ReadLine();
}
/// <summary>
/// Adds all the requests to the package.
/// </summary>
/// <param name="requestHandler">The request Handler</param>
/// <param name="childObjects">The requests to add.</param>
/// <param name="quota">The remaining quota.</param>
/// <returns>The added requests.</returns>
public LinkedList<RCRequest> Pack(RemoteRequestHandler requestHandler, LinkedList<RCRequest> childObjects, ref long quota)
{
LinkedList<RCRequest> addedObjects = new LinkedList<RCRequest>();
// add files that were completed to the package
foreach (RCRequest childObject in childObjects)
{
// add to the package
if (Pack(requestHandler, childObject, ref quota))
{
addedObjects.AddLast(childObject);
}
}
return addedObjects;
}
/// <summary>
/// Adds a RCRequest to the package given the quota limitation.
/// Only called by the remote proxy.
/// XXX: quota is currently simple algorithm.
/// XXX: should be changed to check for compressed size rather than actual size.
/// </summary>
/// <param name="requestHandler">Calling handler for this method.</param>
/// <param name="requestObject">RCRequest to add.</param>
/// <param name="quota">Quota limit.</param>
/// <returns>True iff the request has been packed successfully.</returns>
public bool Pack(RemoteRequestHandler requestHandler, RCRequest requestObject, ref long quota)
{
if (_rcRequests.Contains(requestObject))
{
requestHandler.Logger.Debug("object exists in package: " + requestObject.Uri);
return false;
}
requestObject.FileSize = Utils.GetFileSize(requestObject.CacheFileName);
if (requestObject.FileSize <= 0)
{
return false;
}
// quota check
if ((quota - requestObject.FileSize) < 0)
{
requestHandler.Logger.Debug("object doesn't fit in quota: " + requestObject.Uri);
return false;
}
_rcRequests.Add(requestObject);
quota -= requestObject.FileSize;
return true;
}