/// <summary> /// Gets the eta for a request in Ruralcafe's queue. /// </summary> private void ServeETARequest() { // Parse parameters NameValueCollection qscoll = Util.ParseHtmlQuery(RequestUri); int userId = Int32.Parse(qscoll.Get("u")); string itemId = qscoll.Get("i"); // find the indexer of the matching request List<LocalRequestHandler> requestHandlers = ((RCLocalProxy)_proxy).GetRequests(userId); if (requestHandlers == null) { SendOkHeaders("text/html"); SendMessage("0"); return; } LocalRequestHandler matchingRequestHandler = new LocalRequestHandler(itemId); int requestIndex = requestHandlers.IndexOf(matchingRequestHandler); if (requestIndex < 0) { SendOkHeaders("text/html"); SendMessage("0"); return; } if (requestHandlers[requestIndex].RequestStatus == (int)RequestHandler.Status.Pending) { SendOkHeaders("text/html"); SendMessage("-1"); return; } string printableEta = requestHandlers[requestIndex].PrintableETA(); SendOkHeaders("text/html"); SendMessage(printableEta); }
/// <summary> /// Unpacks the package contents and indexes them. /// </summary> /// <param name="indexPath">Path to the index.</param> /// <param name="requestHandler">Calling handler for this method.</param> /// <returns>Total unpacked content size.</returns> public static long Unpack(LocalRequestHandler requestHandler, string indexPath) { string packageIndexSizeStr = requestHandler.RCRequest.GenericWebResponse.GetResponseHeader("Package-IndexSize"); string packageContentSizeStr = requestHandler.RCRequest.GenericWebResponse.GetResponseHeader("Package-ContentSize"); long packageIndexSize = Int64.Parse(packageIndexSizeStr); long packageContentSize = Int64.Parse(packageContentSizeStr); string packageFileName = requestHandler.PackageFileName; string unpackedPackageFileName = packageFileName.Replace(".gzip", ""); GZipWrapper.GZipDecompress(packageFileName, unpackedPackageFileName, packageIndexSize + packageContentSize); FileStream packageFs = new FileStream(unpackedPackageFileName, FileMode.Open); // read the package index Byte[] packageIndexBuffer = new Byte[packageIndexSize]; packageFs.Read(packageIndexBuffer, 0, (int)packageIndexSize); // split the big package file into pieces string[] stringSeparator = new string[] { "\r\n" }; System.Text.UTF8Encoding enc = new System.Text.UTF8Encoding(); string package = enc.GetString(packageIndexBuffer); string[] packageContentArr = package.Split(stringSeparator, StringSplitOptions.RemoveEmptyEntries); Byte[] bufferOverflow = new Byte[1024]; int bufferOverflowCount = 0; int bytesRead = 0; long bytesReadOfCurrFile = 0; long unpackedBytes = 0; Byte[] buffer = new Byte[1024]; string[] packageEntryArr; string currUri = ""; long currFileSize = 0; foreach (string entry in packageContentArr) { stringSeparator = new string[] { " " }; packageEntryArr = entry.Split(stringSeparator, StringSplitOptions.RemoveEmptyEntries); currUri = packageEntryArr[0]; try { currFileSize = Int64.Parse(packageEntryArr[1]); } catch (Exception e) { requestHandler.LogDebug("problem unpacking: " + entry + " " + e.StackTrace + " " + e.Message); return unpackedBytes; } if (!Util.IsValidUri(currUri)) { requestHandler.LogDebug("problem unpacking: " + currUri); return unpackedBytes; } RCRequest rcRequest = new RCRequest(requestHandler, currUri); unpackedBytes += currFileSize; //requestHandler.LogDebug("unpacking: " + rcRequest.Uri + " - " + currFileSize + " bytes"); // make sure the file doesn't already exist for indexing purposes only bool existed = false; FileInfo ftest = new FileInfo(rcRequest.CacheFileName); if (ftest.Exists) { existed = true; } // try to delete the old version if (!Util.DeleteFile(rcRequest.CacheFileName)) { return unpackedBytes; } // create directory if it doesn't exist if (!Util.CreateDirectoryForFile(rcRequest.CacheFileName)) { return unpackedBytes; } // create the file if it doesn't exist FileStream currFileFS = Util.CreateFile(rcRequest.CacheFileName); if (currFileFS == null) { return unpackedBytes; } // check for overflow from previous file, and use it up first if (bufferOverflowCount > 0) { Buffer.BlockCopy(bufferOverflow, 0, buffer, 0, bufferOverflowCount); bytesRead = bufferOverflowCount; } else { bytesRead = packageFs.Read(buffer, 0, 1024); } // reset for current file bytesReadOfCurrFile = 0; while (bytesRead != 0 && bytesReadOfCurrFile < currFileSize) { // check if we read too much if (bytesReadOfCurrFile + bytesRead > currFileSize) { // bytes left must be less than 1024, fine to convert to Int int bytesLeftOfCurrFile = ((int)(currFileSize - bytesReadOfCurrFile)); currFileFS.Write(buffer, 0, bytesLeftOfCurrFile); // done with this file bytesReadOfCurrFile = currFileSize; // handle overflow bufferOverflowCount = bytesRead - bytesLeftOfCurrFile; Buffer.BlockCopy(buffer, bytesLeftOfCurrFile, bufferOverflow, 0, bytesRead - bytesLeftOfCurrFile); } else { // append what we read currFileFS.Write(buffer, 0, bytesRead); // update bytesReadOfCurrFile bytesReadOfCurrFile += bytesRead; bytesRead = packageFs.Read(buffer, 0, 1024); } } if (bytesReadOfCurrFile != currFileSize) { // ran out of bytes for this file requestHandler.LogDebug("error, unexpected package size: " + rcRequest.CacheFileName + "(" + bytesReadOfCurrFile + " / " + currFileSize + ")"); return unpackedBytes * -1; } currFileFS.Close(); // add the file to Lucene if (Util.IsParseable(rcRequest)) { string document = Util.ReadFileAsString(rcRequest.CacheFileName); string title = Util.GetPageTitle(document); string content = Util.GetPageContent(document); //request.LogDebug("indexing: " + rcRequest._uri); if (!existed) { IndexWrapper.IndexDocument(indexPath, "Content-Type: text/html", rcRequest.Uri, title, content); } } } if (packageFs != null) { packageFs.Close(); } return unpackedBytes; }
/// <summary> /// Removes the request from Ruralcafe's queue. /// </summary> private void RemoveRequest() { // Parse parameters NameValueCollection qscoll = Util.ParseHtmlQuery(RequestUri); int userId = Int32.Parse(qscoll.Get("u")); string itemId = qscoll.Get("i"); LocalRequestHandler matchingRequestHandler = new LocalRequestHandler(itemId); ((RCLocalProxy)_proxy).DequeueRequest(userId, matchingRequestHandler); SendOkHeaders("text/html"); }
/// <summary> /// Read log from directory and add to the requests queue, update the itemId. /// Called upon LocalProxy initialization. /// </summary> /// <param name="logPath">Relative or absolute path for the logs.</param> private bool ReadLog(string logPath) { int highestRequestId = 1; Dictionary<string, List<string>> loggedRequestQueueMap = new Dictionary<string, List<string>>(); string debugFile = DateTime.Now.ToString("s") + "-debug.log"; debugFile = debugFile.Replace(':', '.'); try { if (!Directory.Exists(logPath)) { return false; } DirectoryInfo directory = new DirectoryInfo(logPath); var files = directory.GetFiles("*messages.log").OrderByDescending(f => f.LastWriteTime); int numFiles = files.Count(); if (numFiles == 1) { return true; } FileInfo currentFile = files.ElementAt(1); FileStream fs = currentFile.OpenRead(); Console.WriteLine("Parsing log: " + currentFile); TextReader tr = new StreamReader(fs); uint linesParsed = 0; string line = tr.ReadLine(); string[] lineTokens; while (line != null) { linesParsed++; //Console.WriteLine("Parsing line: " + line); lineTokens = line.Split(' '); string requestId = ""; if (lineTokens.Length > 0) { try { requestId = lineTokens[0]; int requestId_i = Int32.Parse(requestId); if (requestId_i > highestRequestId) { highestRequestId = requestId_i; } } catch (Exception e) { // do nothing } } // maximum number of tokens is 100 if (lineTokens.Length >= 100 || lineTokens.Length <= 5) { //Console.WriteLine("Error, tokens do not fit in array, line " + linesParsed); // read the next line line = tr.ReadLine(); continue; } if (lineTokens.Length >= 9) { // make sure that its actually a search query string clientAddress = lineTokens[4]; string httpCommand = lineTokens[5]; string requestUri = lineTokens[6]; string refererUri = lineTokens[8]; string startTime = lineTokens[1] + " " + lineTokens[2] + " " + lineTokens[3]; if ((httpCommand == "GET") && requestUri.StartsWith("http://www.ruralcafe.net/request/add?")) { // Parse parameters NameValueCollection qscoll = Util.ParseHtmlQuery(requestUri); string targetUri = qscoll.Get("a"); if (targetUri == null) { // error //SendErrorPage(HTTP_NOT_FOUND, "malformed add request", ""); line = tr.ReadLine(); continue; } string fileName = RCRequest.UriToFilePath(targetUri); string hashPath = RCRequest.GetHashPath(fileName); string itemId = hashPath.Replace(Path.DirectorySeparatorChar.ToString(), ""); // add it to the queue //Console.WriteLine("Adding to queue: " + targetUri); List<string> logEntry = new List<string>(); logEntry.Add(requestId); logEntry.Add(startTime); logEntry.Add(clientAddress); logEntry.Add(requestUri); logEntry.Add(refererUri); if (!loggedRequestQueueMap.ContainsKey(itemId)) { loggedRequestQueueMap.Add(itemId, logEntry); } } if ((httpCommand == "GET") && requestUri.StartsWith("http://www.ruralcafe.net/request/remove?")) { // Parse parameters NameValueCollection qscoll = Util.ParseHtmlQuery(requestUri); string itemId = qscoll.Get("i"); if (itemId == null) { // error //SendErrorPage(HTTP_NOT_FOUND, "malformed add request", ""); line = tr.ReadLine(); continue; } // remove it from the queue //Console.WriteLine("Removing from queue: " + itemId); if (loggedRequestQueueMap.ContainsKey(itemId)) { loggedRequestQueueMap.Remove(itemId); } } } else if (lineTokens.Length >= 7) { requestId = lineTokens[0]; string httpCommand = lineTokens[4]; string requestUri = lineTokens[5]; string status = lineTokens[6]; // Parse parameters NameValueCollection qscoll = Util.ParseHtmlQuery(requestUri); string targetUri = qscoll.Get("a"); if (targetUri == null) { // error //SendErrorPage(HTTP_NOT_FOUND, "malformed add request", ""); line = tr.ReadLine(); continue; } string fileName = RCRequest.UriToFilePath(targetUri); string hashPath = RCRequest.GetHashPath(fileName); string itemId = hashPath.Replace(Path.DirectorySeparatorChar.ToString(), ""); if ((httpCommand == "RSP") && loggedRequestQueueMap.ContainsKey(itemId) && Int32.Parse(status) != (int)RequestHandler.Status.Pending) { // parse the response // check if its in the queue, if so, remove it //Console.WriteLine("Removing from queue: " + targetUri); loggedRequestQueueMap[itemId].Add(status); } } // read the next line line = tr.ReadLine(); } tr.Close(); // load the queued requests into the request queue foreach (string currentRequestId in loggedRequestQueueMap.Keys) { LocalRequestHandler requestHandler = new LocalRequestHandler(this, null); requestHandler.HandleLogRequest(loggedRequestQueueMap[currentRequestId]); } // update the nextId NextRequestId = highestRequestId + 1; } catch (Exception e) { Console.WriteLine("Could not read debug logs for saved state."); return false; } return true; }
/// <summary> /// Predicate method for findindex in ETA. /// </summary> /// <param name="requestHandler">The other request object's handler.</param> /// <returns>True or false for match or no match.</returns> private bool SameRCRequestPredicate(LocalRequestHandler requestHandler) { if (_predicateObj == null || requestHandler == null) { return false; } if (_predicateObj.RequestUri == requestHandler.RequestUri) { return true; } return false; }
/// <summary> /// Adds the request to the global queue and client's queue. /// </summary> /// <param name="userId">The IP address of the client.</param> /// <param name="requestHandler">The request to queue.</param> public void QueueRequest(int userId, LocalRequestHandler requestHandler) { // if the request is already completed (due to HandleLogRequest) then don't add to global queue if (!(requestHandler.RequestStatus == (int)RequestHandler.Status.Completed)) { // add the request to the global queue lock (_globalRequestQueue) { if (_globalRequestQueue.Contains(requestHandler)) { // grab the existing handler instead of the new one int existingRequestIndex = _globalRequestQueue.IndexOf(requestHandler); requestHandler = _globalRequestQueue[existingRequestIndex]; } else { // queue new request _globalRequestQueue.Add(requestHandler); _newRequestEvent.Set(); } } } List<LocalRequestHandler> requestHandlers = null; // add the request to the client's queue lock (_clientRequestQueueMap) { if (_clientRequestQueueMap.ContainsKey(userId)) { // get the queue of client requests requestHandlers = _clientRequestQueueMap[userId]; } else { // create the queue of client requests requestHandlers = new List<LocalRequestHandler>(); _clientRequestQueueMap.Add(userId, requestHandlers); } } // add the request to the client's queue if (requestHandlers != null) { lock (requestHandlers) { // add or replace if (requestHandlers.Contains(requestHandler)) { requestHandlers.Remove(requestHandler); requestHandler.OutstandingRequests = requestHandler.OutstandingRequests - 1; } requestHandlers.Add(requestHandler); requestHandler.OutstandingRequests = requestHandler.OutstandingRequests + 1; } } }
/// <summary> /// Starts the listener for connections from clients. /// </summary> public override void StartListener() { WriteDebug("Started Listener on " + _listenAddress + ":" + _listenPort); try { // create a listener for the proxy port TcpListener sockServer = new TcpListener(_listenAddress, _listenPort); sockServer.Start(); // loop and listen for the next connection request while (true) { while (_activeRequests >= MAXIMUM_ACTIVE_REQUESTS) { Thread.Sleep(100); } // accept connections on the proxy port (blocks) Socket socket = sockServer.AcceptSocket(); // handle the accepted connection in a separate thread LocalRequestHandler requestHandler = new LocalRequestHandler(this, socket); // Start own method StartRequestHandler in the thread, which also decreases _activeRequests Thread proxyThread = new Thread(new ParameterizedThreadStart(this.StartRequestHandler)); //proxyThread.Name = String.Format("LocalRequest" + socket.RemoteEndPoint.ToString()); proxyThread.Start(requestHandler); } } catch (SocketException ex) { WriteDebug("SocketException in StartLocalListener, errorcode: " + ex.NativeErrorCode); } catch (IOException e) { WriteDebug("Exception in StartLocalListener: " + e.StackTrace + " " + e.Message); } }
/* // XXX: obsolete /// <summary> /// Removes all requests for a client from the queues. /// </summary> /// <param name="userId">The user Id to remove all requests for.</param> public void ClearRequestQueues(int userId) { // remove from the global queue lock (_globalRequestQueue) { if (_clientRequestQueueMap.ContainsKey(userId)) { List<LocalRequestHandler> requestHandlers = _clientRequestQueueMap[userId]; foreach (LocalRequestHandler requestHandler in requestHandlers) { // check to see if this URI is requested more than once // if so, just decrement count // if not, remove it if (requestHandler.OutstandingRequests == 1) { _globalRequestQueue.Remove(requestHandler); } else { requestHandler.OutstandingRequests = requestHandler.OutstandingRequests - 1; requestHandlers.Remove(requestHandler); } } } } // remove the client address' request queue map. lock (_clientRequestQueueMap) { if (_clientRequestQueueMap.ContainsKey(userId)) { // lock to prevent any additions to the clientRequests List<LocalRequestHandler> requestHandlers = _clientRequestQueueMap[userId]; lock (requestHandlers) { _clientRequestQueueMap.Remove(userId); } } } } */ /// <summary> /// Removes a single request from the queues. /// </summary> /// <param name="userId">The userId of the client.</param> /// <param name="requestHandler">The request to dequeue.</param> public void DequeueRequest(int userId, LocalRequestHandler requestHandler) { // remove the request from the global queue lock (_globalRequestQueue) { if (_globalRequestQueue.Contains(requestHandler)) { int existingRequestIndex = _globalRequestQueue.IndexOf(requestHandler); requestHandler = _globalRequestQueue[existingRequestIndex]; // check to see if this URI is requested more than once // if so, just decrement count // if not, remove it if (requestHandler.OutstandingRequests == 1) { _globalRequestQueue.Remove(requestHandler); } else { //requestHandler.OutstandingRequests = requestHandler.OutstandingRequests - 1; } } } // remove the request from the client's queue // don't need to lock the _clientRequestQueueMap for reading if (_clientRequestQueueMap.ContainsKey(userId)) { List<LocalRequestHandler> requestHandlers = _clientRequestQueueMap[userId]; lock (requestHandlers) { if (requestHandlers.Contains(requestHandler)) { requestHandler.OutstandingRequests = requestHandler.OutstandingRequests - 1; requestHandlers.Remove(requestHandler); } } } }
/// <summary> /// Returns the number of seconds until request is expected to be satisfied. /// Calculates the ETA by looking at the average satisfied time and the position of this request. /// </summary> /// <param name="requestHandler">The request for which we want the ETA.</param> /// <returns>ETA in seconds.</returns> public int ETA(LocalRequestHandler requestHandler) { // set the predicate object _predicateObj = requestHandler; int requestPosition = _globalRequestQueue.FindIndex(SameRCRequestPredicate); // +2 since -1 if the item doesn't exist (which means its being serviced now) return (int)((requestPosition + 2) * _averageTimePerRequest.TotalSeconds); }
/// <summary> /// Unpacks the package contents and indexes them. Throws an Exception if anything goes wrong. /// </summary> /// <param name="requestHandler">Calling handler for this method.</param> /// <param name="rcheaders">The rc specific headers.</param> /// <returns>Total unpacked content size.</returns> public static long Unpack(LocalRequestHandler requestHandler, RCSpecificResponseHeaders rcheaders) { long packageIndexSize = rcheaders.RCPackageIndexSize; long packageContentSize = rcheaders.RCPackageContentSize; if (packageIndexSize == 0 || packageContentSize == 0) { // This is an internal error that should not happen! throw new Exception("problem unpacking: package index or content size is 0."); } string packageFileName = requestHandler.PackageFileName; string unpackedPackageFileName = packageFileName.Replace(".gzip", ""); GZipWrapper.GZipDecompress(packageFileName, unpackedPackageFileName, packageIndexSize + packageContentSize); FileStream packageFs = new FileStream(unpackedPackageFileName, FileMode.Open); // read the package index Byte[] packageIndexBuffer = new Byte[packageIndexSize]; int bytesOfIndexRead = 0; while (bytesOfIndexRead != packageIndexSize) { int read = packageFs.Read(packageIndexBuffer, bytesOfIndexRead, (int)(packageIndexSize - bytesOfIndexRead)); if(read == 0) { // This should not happen throw new Exception("problem unpacking: could not read index."); } bytesOfIndexRead += read; } // split the big package file into pieces string[] stringSeparator = new string[] { "\r\n" }; System.Text.UTF8Encoding enc = new System.Text.UTF8Encoding(); string package = enc.GetString(packageIndexBuffer); string[] packageContentArr = package.Split(stringSeparator, StringSplitOptions.RemoveEmptyEntries); long unpackedBytes = 0; HashSet<GlobalCacheItemToAdd> itemsToAdd = new HashSet<GlobalCacheItemToAdd>(); try { for (int i = 0; i < packageContentArr.Length; i += 2) { // Index format is 2 lines: // <statusCode> <fileSize> <filename> (filename is last, as it can have spaces) // <headers> (JSON) string[] firstLineArray = packageContentArr[i].Split(new string[] { " " }, 3, StringSplitOptions.None); if (firstLineArray.Length != 3) { throw new Exception("unparseable entry: " + packageContentArr[i]); } short statusCode; long fileSize; try { statusCode = Int16.Parse(firstLineArray[0]); fileSize = Int64.Parse(firstLineArray[1]); } catch (Exception) { throw new Exception("unparseable entry: " + packageContentArr[i]); } string fileName = firstLineArray[2]; string headersJson = packageContentArr[i + 1]; NameValueCollection headers = JsonConvert.DeserializeObject<NameValueCollection>(headersJson, new NameValueCollectionConverter()); if (requestHandler.Proxy.ProxyCacheManager.CreateOrUpdateFileAndWrite(fileName, fileSize, packageFs)) { unpackedBytes += fileSize; // We index only if (i==0), which means only the first file of the package. This is the main // page that has been requested. Embedded pages won't be indexed. GlobalCacheItemToAdd newItem = new GlobalCacheItemToAdd(fileName, headers, statusCode, i == 0); itemsToAdd.Add(newItem); } } } finally { if (packageFs != null) { packageFs.Close(); } // Add all Database entries if (!requestHandler.Proxy.ProxyCacheManager.AddCacheItemsForExistingFiles(itemsToAdd)) { // Adding to the DB failed throw new Exception("Adding an item to the database failed."); } } return unpackedBytes; }
/// <summary> /// Adds the request handler to the user's queue, no dups. /// </summary> /// <param name="userId">The user's id.</param> /// <param name="requestHandler">The request handler to queue.</param> private void AddRequestUserQueue(int userId, LocalRequestHandler requestHandler) { List<LocalRequestHandler> requestHandlers = null; // add client queue, if it does not exist yet lock (_clientRequestsMap) { if (_clientRequestsMap.ContainsKey(userId)) { // get the queue of client requests requestHandlers = _clientRequestsMap[userId]; } else { // create the queue of client requests requestHandlers = new List<LocalRequestHandler>(); _clientRequestsMap.Add(userId, requestHandlers); } } // add the request to the client's queue, if not contained already. lock (requestHandlers) { if (!requestHandlers.Contains(requestHandler)) { requestHandlers.Add(requestHandler); requestHandler.OutstandingRequests++; } } }
/// <summary> /// Returns the number of seconds until request is expected to be satisfied. /// Calculates the ETA by looking at the average satisfied time, the current running time of /// requests and the position of this request. /// </summary> /// <param name="requestHandler">The request for which we want the ETA.</param> /// <returns>ETA in seconds.</returns> public int ETA(LocalRequestHandler requestHandler) { double avg = _averageTimePerRequest.TotalSeconds; int pos = _globalRequests.IndexOf(requestHandler); if (requestHandler.RequestStatus == RequestHandler.Status.Downloading) { // Calculate remaining time return (int)(avg - Math.Min(requestHandler.TimeRunning().TotalSeconds, avg)); } else if (requestHandler.RequestStatus == RequestHandler.Status.Pending) { // Determine request that is running longest. TimeSpan max = TimeSpan.Zero; for (int i = 0; i < _maxInflightRequests && i < _globalRequests.Count; i++) { TimeSpan ts; if ((ts = _globalRequests[i].TimeRunning()) > max) { max = ts; } } // Calculate remaining time return (int)((pos + 1 - _maxInflightRequests) * avg - Math.Min(max.TotalSeconds, avg)); } else // finished or error { return 0; } }
/// <summary> /// Adds a request, where the user is unknown yet, to the Dictionary /// </summary> /// <param name="handler">The value.</param> /// <returns>The id of the request.</returns> public int AddRequestWithoutUser(LocalRequestHandler handler) { int id = _random.Next(); lock (_requestsWithoutUser) { if (_requestsWithoutUser.Count >= REQUESTS_WITHOUT_USER_CAPACITY) { _requestsWithoutUser.RemoveAt(0); } _requestsWithoutUser.Add(new KeyValuePair<int,LocalRequestHandler>(id, handler)); } return id; }
/// <summary> /// Adds the request to the global queue and client's queue and wakes up the dispatcher. /// </summary> /// <param name="userId">The user's id.</param> /// <param name="requestHandler">The request handler to queue.</param> public void AddRequest(int userId, LocalRequestHandler requestHandler) { // Order is important! requestHandler = (LocalRequestHandler) AddRequestGlobalQueue(requestHandler); AddRequestUserQueue(userId, requestHandler); // Notify that a new request has been added. The Dispatcher will wake up if it was waiting. _requestEvent.Set(); }