public bool TryOrderedDequeue(out EntityUpdate value, out Int32 timeinqueue)
{
MinHeap <MinHeapItem> curheap;
for (int iq = 0; iq < NumberOfQueues; ++iq)
{
curheap = m_heaps[iq];
if (curheap.Count > 0)
{
MinHeapItem item = curheap.RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
}
timeinqueue = 0;
value = default(EntityUpdate);
if (m_lookupTable.Count == 0 && m_added > 8 * m_capacity)
{
m_lookupTable = new Dictionary <uint, LookupItem>(m_capacity);
m_added = 0;
}
return(false);
}
public BasicDOSProtector(BasicDosProtectorOptions options)
{
_generalRequestTimes = new CircularBuffer <int>(options.MaxRequestsInTimeframe + 1, true);
_generalRequestTimes.Put(0);
_options = options;
_deeperInspection = new Dictionary <string, CircularBuffer <int> >();
_tempBlocked = new Dictionary <string, int>();
_sessions = new Dictionary <string, int>();
_forgetTimer = new System.Timers.Timer();
_forgetTimer.Elapsed += delegate
{
_forgetTimer.Enabled = false;
List <string> removes = new List <string>();
_blockLockSlim.EnterReadLock();
foreach (string str in _tempBlocked.Keys)
{
if (
Util.EnvironmentTickCountSubtract(Util.EnvironmentTickCount(),
_tempBlocked[str]) > 0)
{
removes.Add(str);
}
}
_blockLockSlim.ExitReadLock();
lock (_deeperInspection)
{
_blockLockSlim.EnterWriteLock();
for (int i = 0; i < removes.Count; i++)
{
_tempBlocked.Remove(removes[i]);
_deeperInspection.Remove(removes[i]);
_sessions.Remove(removes[i]);
}
_blockLockSlim.ExitWriteLock();
}
foreach (string str in removes)
{
m_log.InfoFormat("[{0}] client: {1} is no longer blocked.",
_options.ReportingName, str);
}
_blockLockSlim.EnterReadLock();
if (_tempBlocked.Count > 0)
{
_forgetTimer.Enabled = true;
}
_blockLockSlim.ExitReadLock();
};
_forgetTimer.Interval = _options.ForgetTimeSpan.TotalMilliseconds;
}
/// <summary>
/// At this point, the rate limiting code needs to track 'per user' velocity.
/// </summary>
/// <param name="key">Context Key, string representing a rate limiting context</param>
/// <param name="endpoint"></param>
/// <returns></returns>
private bool DeeperInspection(string key, string endpoint)
{
lock (_deeperInspection)
{
string clientstring = key;
if (_deeperInspection.ContainsKey(clientstring))
{
_deeperInspection[clientstring].Put(Util.EnvironmentTickCount());
if (_deeperInspection[clientstring].Size == _deeperInspection[clientstring].Capacity &&
(Util.EnvironmentTickCountSubtract(Util.EnvironmentTickCount(), _deeperInspection[clientstring].Get()) <
_options.RequestTimeSpan.TotalMilliseconds))
{
//Looks like we're over the limit
_blockLockSlim.EnterWriteLock();
if (!_tempBlocked.ContainsKey(clientstring))
{
_tempBlocked.Add(clientstring, Util.EnvironmentTickCount() + (int)_options.ForgetTimeSpan.TotalMilliseconds);
}
else
{
_tempBlocked[clientstring] = Util.EnvironmentTickCount() + (int)_options.ForgetTimeSpan.TotalMilliseconds;
}
_blockLockSlim.ExitWriteLock();
m_log.WarnFormat("[{0}]: client: {1} is blocked for {2} milliseconds, X-ForwardedForAllowed status is {3}, endpoint:{4}", _options.ReportingName, clientstring, _options.ForgetTimeSpan.TotalMilliseconds, _options.AllowXForwardedFor, endpoint);
return(false);
}
//else
// return true;
}
else
{
_deeperInspection.Add(clientstring, new CircularBuffer <int>(_options.MaxRequestsInTimeframe + 1, true));
_deeperInspection[clientstring].Put(Util.EnvironmentTickCount());
_forgetTimer.Enabled = true;
}
}
return(true);
}
public bool TryOrderedDequeue(out EntityUpdate value, out Int32 timeinqueue)
{
// If there is anything in imediate queues, return it first no
// matter what else. Breaks fairness. But very useful.
for (int iq = 0; iq < NumberOfQueues; iq++)
{
if (m_heaps[iq].Count > 0)
{
MinHeapItem item = m_heaps[iq].RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
}
timeinqueue = 0;
value = default(EntityUpdate);
return(false);
}
/// <summary>
/// Remove an item from one of the queues. Specifically, it removes the
/// oldest item from the next queue in order to provide fair access to
/// all of the queues
/// </summary>
public bool TryDequeue(out EntityUpdate value, out Int32 timeinqueue)
{
// If there is anything in immediate queues, return it first no
// matter what else. Breaks fairness. But very useful.
for (int iq = 0; iq < NumberOfImmediateQueues; iq++)
{
if (m_heaps[iq].Count > 0)
{
MinHeapItem item = m_heaps[iq].RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
}
// To get the fair queing, we cycle through each of the
// queues when finding an element to dequeue.
// We pull (NumberOfQueues - QueueIndex) items from each queue in order
// to give lower numbered queues a higher priority and higher percentage
// of the bandwidth.
MinHeap <MinHeapItem> curheap = m_heaps[m_nextQueue];
// Check for more items to be pulled from the current queue
if (m_countFromQueue > 0 && curheap.Count > 0)
{
--m_countFromQueue;
MinHeapItem item = curheap.RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
// Find the next non-immediate queue with updates in it
for (uint i = NumberOfImmediateQueues; i < NumberOfQueues; ++i)
{
m_nextQueue++;
if (m_nextQueue >= NumberOfQueues)
{
m_nextQueue = NumberOfImmediateQueues;
}
curheap = m_heaps[m_nextQueue];
if (curheap.Count == 0)
{
continue;
}
m_countFromQueue = m_queueCounts[m_nextQueue];
--m_countFromQueue;
MinHeapItem item = curheap.RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
timeinqueue = 0;
value = default(EntityUpdate);
if (m_lookupTable.Count == 0 && m_added > 8 * m_capacity)
{
m_lookupTable = new Dictionary <uint, LookupItem>(m_capacity);
m_added = 0;
}
return(false);
}
/// <summary>
/// Perform a synchronous REST request.
/// </summary>
/// <param name="verb"></param>
/// <param name="requestUrl"></param>
/// <param name="obj"> </param>
/// <returns></returns>
///
/// <exception cref="System.Net.WebException">Thrown if we encounter a network issue while posting
/// the request. You'll want to make sure you deal with this as they're not uncommon</exception>
public static string MakeRequest(string verb, string requestUrl, string obj)
{
int reqnum = WebUtil.RequestNumber++;
if (WebUtil.DebugLevel >= 3)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} SynchronousRestForms {1} {2}",
reqnum, verb, requestUrl);
}
int tickstart = Util.EnvironmentTickCount();
int tickdata = 0;
WebRequest request = WebRequest.Create(requestUrl);
request.Method = verb;
string respstring = String.Empty;
using (MemoryStream buffer = new MemoryStream())
{
if ((verb == "POST") || (verb == "PUT"))
{
request.ContentType = "application/x-www-form-urlencoded";
int length = 0;
using (StreamWriter writer = new StreamWriter(buffer))
{
writer.Write(obj);
writer.Flush();
}
length = (int)obj.Length;
request.ContentLength = length;
Stream requestStream = null;
try
{
requestStream = request.GetRequestStream();
requestStream.Write(buffer.ToArray(), 0, length);
}
catch (Exception e)
{
m_log.DebugFormat(
"[FORMS]: exception occured {0} {1}: {2}{3}", verb, requestUrl, e.Message, e.StackTrace);
}
finally
{
if (requestStream != null)
{
requestStream.Close();
}
// capture how much time was spent writing
tickdata = Util.EnvironmentTickCountSubtract(tickstart);
}
}
try
{
using (WebResponse resp = request.GetResponse())
{
if (resp.ContentLength != 0)
{
Stream respStream = null;
try
{
respStream = resp.GetResponseStream();
using (StreamReader reader = new StreamReader(respStream))
{
respstring = reader.ReadToEnd();
}
}
catch (Exception e)
{
m_log.DebugFormat(
"[FORMS]: Exception occured on receiving {0} {1}: {2}{3}",
verb, requestUrl, e.Message, e.StackTrace);
}
finally
{
if (respStream != null)
{
respStream.Close();
}
}
}
}
}
catch (System.InvalidOperationException)
{
// This is what happens when there is invalid XML
m_log.DebugFormat("[FORMS]: InvalidOperationException on receiving {0} {1}", verb, requestUrl);
}
}
int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
if (tickdiff > WebUtil.LongCallTime)
{
m_log.InfoFormat(
"[FORMS]: Slow request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
reqnum,
verb,
requestUrl,
tickdiff,
tickdata,
obj.Length > WebUtil.MaxRequestDiagLength ? obj.Remove(WebUtil.MaxRequestDiagLength) : obj);
}
else if (WebUtil.DebugLevel >= 4)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
reqnum, tickdiff, tickdata);
}
return(respstring);
}
/// <summary>
/// Perform an asynchronous REST request.
/// </summary>
/// <param name="verb">GET or POST</param>
/// <param name="requestUrl"></param>
/// <param name="obj"></param>
/// <param name="action"></param>
/// <returns></returns>
///
/// <exception cref="System.Net.WebException">Thrown if we encounter a
/// network issue while posting the request. You'll want to make
/// sure you deal with this as they're not uncommon</exception>
//
public static void MakeRequest <TRequest, TResponse>(string verb,
string requestUrl, TRequest obj, Action <TResponse> action)
{
int reqnum = WebUtil.RequestNumber++;
if (WebUtil.DebugLevel >= 3)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} AsynchronousRequestObject {1} {2}",
reqnum, verb, requestUrl);
}
int tickstart = Util.EnvironmentTickCount();
int tickdata = 0;
Type type = typeof(TRequest);
WebRequest request = WebRequest.Create(requestUrl);
WebResponse response = null;
TResponse deserial = default(TResponse);
XmlSerializer deserializer = new XmlSerializer(typeof(TResponse));
request.Method = verb;
MemoryStream buffer = null;
if (verb == "POST")
{
request.ContentType = "text/xml";
buffer = new MemoryStream();
XmlWriterSettings settings = new XmlWriterSettings();
settings.Encoding = Encoding.UTF8;
using (XmlWriter writer = XmlWriter.Create(buffer, settings))
{
XmlSerializer serializer = new XmlSerializer(type);
serializer.Serialize(writer, obj);
writer.Flush();
}
int length = (int)buffer.Length;
request.ContentLength = length;
request.BeginGetRequestStream(delegate(IAsyncResult res)
{
Stream requestStream = request.EndGetRequestStream(res);
requestStream.Write(buffer.ToArray(), 0, length);
requestStream.Close();
// capture how much time was spent writing
tickdata = Util.EnvironmentTickCountSubtract(tickstart);
request.BeginGetResponse(delegate(IAsyncResult ar)
{
response = request.EndGetResponse(ar);
Stream respStream = null;
try
{
respStream = response.GetResponseStream();
deserial = (TResponse)deserializer.Deserialize(
respStream);
}
catch (System.InvalidOperationException)
{
}
finally
{
// Let's not close this
//buffer.Close();
respStream.Close();
response.Close();
}
action(deserial);
}, null);
}, null);
}
else
{
request.BeginGetResponse(delegate(IAsyncResult res2)
{
try
{
// If the server returns a 404, this appears to trigger a System.Net.WebException even though that isn't
// documented in MSDN
response = request.EndGetResponse(res2);
Stream respStream = null;
try
{
respStream = response.GetResponseStream();
deserial = (TResponse)deserializer.Deserialize(respStream);
}
catch (System.InvalidOperationException)
{
}
finally
{
respStream.Close();
response.Close();
}
}
catch (WebException e)
{
if (e.Status == WebExceptionStatus.ProtocolError)
{
if (e.Response is HttpWebResponse)
{
HttpWebResponse httpResponse = (HttpWebResponse)e.Response;
if (httpResponse.StatusCode != HttpStatusCode.NotFound)
{
// We don't appear to be handling any other status codes, so log these feailures to that
// people don't spend unnecessary hours hunting phantom bugs.
m_log.DebugFormat(
"[ASYNC REQUEST]: Request {0} {1} failed with unexpected status code {2}",
verb, requestUrl, httpResponse.StatusCode);
}
}
}
else
{
m_log.ErrorFormat(
"[ASYNC REQUEST]: Request {0} {1} failed with status {2} and message {3}",
verb, requestUrl, e.Status, e.Message);
}
}
catch (Exception e)
{
m_log.ErrorFormat(
"[ASYNC REQUEST]: Request {0} {1} failed with exception {2}{3}",
verb, requestUrl, e.Message, e.StackTrace);
}
// m_log.DebugFormat("[ASYNC REQUEST]: Received {0}", deserial.ToString());
try
{
action(deserial);
}
catch (Exception e)
{
m_log.ErrorFormat(
"[ASYNC REQUEST]: Request {0} {1} callback failed with exception {2}{3}",
verb, requestUrl, e.Message, e.StackTrace);
}
}, null);
}
int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
if (tickdiff > WebUtil.LongCallTime)
{
string originalRequest = null;
if (buffer != null)
{
originalRequest = Encoding.UTF8.GetString(buffer.ToArray());
if (originalRequest.Length > WebUtil.MaxRequestDiagLength)
{
originalRequest = originalRequest.Remove(WebUtil.MaxRequestDiagLength);
}
}
m_log.InfoFormat(
"[ASYNC REQUEST]: Slow request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
reqnum,
verb,
requestUrl,
tickdiff,
tickdata,
originalRequest);
}
else if (WebUtil.DebugLevel >= 4)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
reqnum, tickdiff, tickdata);
}
}
private static OSDMap ServiceFormRequestWorker(string url, NameValueCollection data, int timeout)
{
int reqnum = RequestNumber++;
string method = (data != null && data["RequestMethod"] != null) ? data["RequestMethod"] : "unknown";
if (DebugLevel >= 3)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} ServiceForm {1} {2} (timeout {3})",
reqnum, method, url, timeout);
}
string errorMessage = "unknown error";
int tickstart = Util.EnvironmentTickCount();
int tickdata = 0;
string queryString = null;
try
{
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
request.Method = "POST";
request.Timeout = timeout;
request.KeepAlive = false;
request.MaximumAutomaticRedirections = 10;
request.ReadWriteTimeout = timeout / 4;
request.Headers[OSHeaderRequestID] = reqnum.ToString();
if (data != null)
{
queryString = BuildQueryString(data);
byte[] buffer = System.Text.Encoding.UTF8.GetBytes(queryString);
request.ContentLength = buffer.Length;
request.ContentType = "application/x-www-form-urlencoded";
using (Stream requestStream = request.GetRequestStream())
requestStream.Write(buffer, 0, buffer.Length);
}
// capture how much time was spent writing, this may seem silly
// but with the number concurrent requests, this often blocks
tickdata = Util.EnvironmentTickCountSubtract(tickstart);
using (WebResponse response = request.GetResponse())
{
using (Stream responseStream = response.GetResponseStream())
{
string responseStr = null;
responseStr = responseStream.GetStreamString();
OSD responseOSD = OSDParser.Deserialize(responseStr);
if (responseOSD.Type == OSDType.Map)
{
return((OSDMap)responseOSD);
}
}
}
}
catch (WebException we)
{
errorMessage = we.Message;
if (we.Status == WebExceptionStatus.ProtocolError)
{
HttpWebResponse webResponse = (HttpWebResponse)we.Response;
errorMessage = String.Format("[{0}] {1}", webResponse.StatusCode, webResponse.StatusDescription);
}
}
catch (Exception ex)
{
errorMessage = ex.Message;
}
finally
{
int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
if (tickdiff > LongCallTime)
{
m_log.InfoFormat(
"[WEB UTIL]: Slow ServiceForm request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
reqnum,
method,
url,
tickdiff,
tickdata,
queryString != null
? (queryString.Length > MaxRequestDiagLength) ? queryString.Remove(MaxRequestDiagLength) : queryString
: "");
}
else if (DebugLevel >= 4)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
reqnum, tickdiff, tickdata);
}
}
m_log.WarnFormat("[WEB UTIL]: ServiceForm request {0} {1} {2} failed: {2}", reqnum, method, url, errorMessage);
return(ErrorResponseMap(errorMessage));
}
private static OSDMap ServiceOSDRequestWorker(string url, OSDMap data, string method, int timeout, bool compressed)
{
int reqnum = RequestNumber++;
if (DebugLevel >= 3)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} ServiceOSD {1} {2} (timeout {3}, compressed {4})",
reqnum, method, url, timeout, compressed);
}
string errorMessage = "unknown error";
int tickstart = Util.EnvironmentTickCount();
int tickdata = 0;
string strBuffer = null;
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.Timeout = timeout;
request.KeepAlive = false;
request.MaximumAutomaticRedirections = 10;
request.ReadWriteTimeout = timeout / 4;
request.Headers[OSHeaderRequestID] = reqnum.ToString();
// If there is some input, write it into the request
if (data != null)
{
strBuffer = OSDParser.SerializeJsonString(data);
byte[] buffer = System.Text.Encoding.UTF8.GetBytes(strBuffer);
if (compressed)
{
request.ContentType = "application/x-gzip";
using (MemoryStream ms = new MemoryStream())
{
using (GZipStream comp = new GZipStream(ms, CompressionMode.Compress))
{
comp.Write(buffer, 0, buffer.Length);
// We need to close the gzip stream before we write it anywhere
// because apparently something important related to gzip compression
// gets written on the strteam upon Dispose()
}
byte[] buf = ms.ToArray();
request.ContentLength = buf.Length; //Count bytes to send
using (Stream requestStream = request.GetRequestStream())
requestStream.Write(buf, 0, (int)buf.Length);
}
}
else
{
request.ContentType = "application/json";
request.ContentLength = buffer.Length; //Count bytes to send
using (Stream requestStream = request.GetRequestStream())
requestStream.Write(buffer, 0, buffer.Length); //Send it
}
}
// capture how much time was spent writing, this may seem silly
// but with the number concurrent requests, this often blocks
tickdata = Util.EnvironmentTickCountSubtract(tickstart);
using (WebResponse response = request.GetResponse())
{
using (Stream responseStream = response.GetResponseStream())
{
string responseStr = null;
responseStr = responseStream.GetStreamString();
// m_log.DebugFormat("[WEB UTIL]: <{0}> response is <{1}>",reqnum,responseStr);
return(CanonicalizeResults(responseStr));
}
}
}
catch (WebException we)
{
errorMessage = we.Message;
if (we.Status == WebExceptionStatus.ProtocolError)
{
HttpWebResponse webResponse = (HttpWebResponse)we.Response;
errorMessage = String.Format("[{0}] {1}", webResponse.StatusCode, webResponse.StatusDescription);
}
}
catch (Exception ex)
{
errorMessage = ex.Message;
}
finally
{
int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
if (tickdiff > LongCallTime)
{
m_log.InfoFormat(
"[WEB UTIL]: Slow ServiceOSD request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
reqnum,
method,
url,
tickdiff,
tickdata,
strBuffer != null
? (strBuffer.Length > MaxRequestDiagLength ? strBuffer.Remove(MaxRequestDiagLength) : strBuffer)
: "");
}
else if (DebugLevel >= 4)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
reqnum, tickdiff, tickdata);
}
}
m_log.DebugFormat(
"[WEB UTIL]: ServiceOSD request {0} {1} {2} FAILED: {3}", reqnum, url, method, errorMessage);
return(ErrorResponseMap(errorMessage));
}
/// <summary>
/// Perform a synchronous REST request.
/// </summary>
/// <param name="verb"></param>
/// <param name="requestUrl"></param>
/// <param name="obj"> </param>
/// <returns></returns>
///
/// <exception cref="System.Net.WebException">Thrown if we encounter a network issue while posting
/// the request. You'll want to make sure you deal with this as they're not uncommon</exception>
public static TResponse MakeRequest <TRequest, TResponse>(string verb, string requestUrl, TRequest obj)
{
int reqnum = WebUtil.RequestNumber++;
if (WebUtil.DebugLevel >= 3)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} SynchronousRestObject {1} {2}",
reqnum, verb, requestUrl);
}
int tickstart = Util.EnvironmentTickCount();
int tickdata = 0;
Type type = typeof(TRequest);
TResponse deserial = default(TResponse);
WebRequest request = WebRequest.Create(requestUrl);
request.Method = verb;
MemoryStream buffer = null;
if ((verb == "POST") || (verb == "PUT"))
{
request.ContentType = "text/xml";
buffer = new MemoryStream();
XmlWriterSettings settings = new XmlWriterSettings();
settings.Encoding = Encoding.UTF8;
using (XmlWriter writer = XmlWriter.Create(buffer, settings))
{
XmlSerializer serializer = new XmlSerializer(type);
serializer.Serialize(writer, obj);
writer.Flush();
}
int length = (int)buffer.Length;
request.ContentLength = length;
Stream requestStream = null;
try
{
requestStream = request.GetRequestStream();
requestStream.Write(buffer.ToArray(), 0, length);
}
catch (Exception e)
{
m_log.DebugFormat(
"[SynchronousRestObjectRequester]: Exception in making request {0} {1}: {2}{3}",
verb, requestUrl, e.Message, e.StackTrace);
return(deserial);
}
finally
{
if (requestStream != null)
{
requestStream.Close();
}
// capture how much time was spent writing
tickdata = Util.EnvironmentTickCountSubtract(tickstart);
}
}
try
{
using (HttpWebResponse resp = (HttpWebResponse)request.GetResponse())
{
if (resp.ContentLength != 0)
{
Stream respStream = resp.GetResponseStream();
XmlSerializer deserializer = new XmlSerializer(typeof(TResponse));
deserial = (TResponse)deserializer.Deserialize(respStream);
respStream.Close();
}
else
{
m_log.DebugFormat(
"[SynchronousRestObjectRequester]: Oops! no content found in response stream from {0} {1}",
verb, requestUrl);
}
}
}
catch (WebException e)
{
HttpWebResponse hwr = (HttpWebResponse)e.Response;
if (hwr != null && hwr.StatusCode == HttpStatusCode.NotFound)
{
return(deserial);
}
else
{
m_log.ErrorFormat(
"[SynchronousRestObjectRequester]: WebException for {0} {1} {2}: {3} {4}",
verb, requestUrl, typeof(TResponse).ToString(), e.Message, e.StackTrace);
}
}
catch (System.InvalidOperationException)
{
// This is what happens when there is invalid XML
m_log.DebugFormat(
"[SynchronousRestObjectRequester]: Invalid XML from {0} {1} {2}",
verb, requestUrl, typeof(TResponse).ToString());
}
catch (Exception e)
{
m_log.DebugFormat(
"[SynchronousRestObjectRequester]: Exception on response from {0} {1}: {2}{3}",
verb, requestUrl, e.Message, e.StackTrace);
}
int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
if (tickdiff > WebUtil.LongCallTime)
{
string originalRequest = null;
if (buffer != null)
{
originalRequest = Encoding.UTF8.GetString(buffer.ToArray());
if (originalRequest.Length > WebUtil.MaxRequestDiagLength)
{
originalRequest = originalRequest.Remove(WebUtil.MaxRequestDiagLength);
}
}
m_log.InfoFormat(
"[SynchronousRestObjectRequester]: Slow request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
reqnum,
verb,
requestUrl,
tickdiff,
tickdata,
originalRequest);
}
else if (WebUtil.DebugLevel >= 4)
{
m_log.DebugFormat(
"[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
reqnum, tickdiff, tickdata);
}
return(deserial);
}
public static OSDMap ServiceOSDRequest(string url, OSDMap data, string method, int timeout)
{
int reqnum = m_requestNumber++;
// m_log.DebugFormat("[WEB UTIL]: <{0}> start osd request for {1}, method {2}",reqnum,url,method);
string errorMessage = "unknown error";
int tickstart = Util.EnvironmentTickCount();
int tickdata = 0;
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = method;
request.Timeout = timeout;
request.KeepAlive = false;
request.MaximumAutomaticRedirections = 10;
request.ReadWriteTimeout = timeout * 8;
request.Headers[OSHeaderRequestID] = reqnum.ToString();
// If there is some input, write it into the request
if (data != null)
{
string strBuffer = OSDParser.SerializeJsonString(data);
byte[] buffer = System.Text.Encoding.UTF8.GetBytes(strBuffer);
request.ContentType = "application/json";
request.ContentLength = buffer.Length; //Count bytes to send
using (Stream requestStream = request.GetRequestStream())
requestStream.Write(buffer, 0, buffer.Length); //Send it
}
// capture how much time was spent writing, this may seem silly
// but with the number concurrent requests, this often blocks
tickdata = Util.EnvironmentTickCountSubtract(tickstart);
using (WebResponse response = request.GetResponse())
{
using (Stream responseStream = response.GetResponseStream())
{
string responseStr = null;
responseStr = responseStream.GetStreamString();
// m_log.DebugFormat("[WEB UTIL]: <{0}> response is <{1}>",reqnum,responseStr);
return(CanonicalizeResults(responseStr));
}
}
}
catch (WebException we)
{
errorMessage = we.Message;
if (we.Status == WebExceptionStatus.ProtocolError)
{
HttpWebResponse webResponse = (HttpWebResponse)we.Response;
errorMessage = String.Format("[{0}] {1}", webResponse.StatusCode, webResponse.StatusDescription);
}
}
catch (Exception ex)
{
errorMessage = ex.Message;
}
finally
{
// This just dumps a warning for any operation that takes more than 100 ms
int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
if (tickdiff > LongCallTime)
{
m_log.InfoFormat("[WEB UTIL]: osd request <{0}> (URI:{1}, METHOD:{2}) took {3}ms overall, {4}ms writing",
reqnum, url, method, tickdiff, tickdata);
}
}
m_log.WarnFormat("[WEB UTIL]: <{0}> osd request for {1}, method {2} FAILED: {3}", reqnum, url, method, errorMessage);
return(ErrorResponseMap(errorMessage));
}
/// <summary>
/// Process the velocity of this context
/// </summary>
/// <param name="key"></param>
/// <param name="endpoint"></param>
/// <returns></returns>
public bool Process(string key, string endpoint)
{
if (_options.MaxRequestsInTimeframe < 1 || _options.RequestTimeSpan.TotalMilliseconds < 1)
{
return(true);
}
string clientstring = key;
_blockLockSlim.EnterReadLock();
if (_tempBlocked.ContainsKey(clientstring))
{
_blockLockSlim.ExitReadLock();
if (_options.ThrottledAction == ThrottleAction.DoThrottledMethod)
{
return(false);
}
else
{
throw new System.Security.SecurityException("Throttled");
}
}
_blockLockSlim.ExitReadLock();
lock (_generalRequestTimes)
_generalRequestTimes.Put(Util.EnvironmentTickCount());
if (_options.MaxConcurrentSessions > 0)
{
int sessionscount = 0;
_sessionLockSlim.EnterReadLock();
if (_sessions.ContainsKey(key))
{
sessionscount = _sessions[key];
}
_sessionLockSlim.ExitReadLock();
if (sessionscount > _options.MaxConcurrentSessions)
{
// Add to blocking and cleanup methods
lock (_deeperInspection)
{
_blockLockSlim.EnterWriteLock();
if (!_tempBlocked.ContainsKey(clientstring))
{
_tempBlocked.Add(clientstring,
Util.EnvironmentTickCount() +
(int)_options.ForgetTimeSpan.TotalMilliseconds);
_forgetTimer.Enabled = true;
m_log.WarnFormat("[{0}]: client: {1} is blocked for {2} milliseconds based on concurrency, X-ForwardedForAllowed status is {3}, endpoint:{4}", _options.ReportingName, clientstring, _options.ForgetTimeSpan.TotalMilliseconds, _options.AllowXForwardedFor, endpoint);
}
else
{
_tempBlocked[clientstring] = Util.EnvironmentTickCount() +
(int)_options.ForgetTimeSpan.TotalMilliseconds;
}
_blockLockSlim.ExitWriteLock();
}
}
else
{
ProcessConcurrency(key, endpoint);
}
}
if (_generalRequestTimes.Size == _generalRequestTimes.Capacity &&
(Util.EnvironmentTickCountSubtract(Util.EnvironmentTickCount(), _generalRequestTimes.Get()) <
_options.RequestTimeSpan.TotalMilliseconds))
{
//Trigger deeper inspection
if (DeeperInspection(key, endpoint))
{
return(true);
}
if (_options.ThrottledAction == ThrottleAction.DoThrottledMethod)
{
return(false);
}
else
{
throw new System.Security.SecurityException("Throttled");
}
}
return(true);
}
/// <summary>
/// Remove an item from one of the queues. Specifically, it removes the
/// oldest item from the next queue in order to provide fair access to
/// all of the queues
/// </summary>
public bool TryDequeue(out IEntityUpdate value, out Int32 timeinqueue)
{
// If there is anything in priority queue 0, return it first no
// matter what else. Breaks fairness. But very useful.
for (int iq = 0; iq < NumberOfImmediateQueues; iq++)
{
if (m_heaps[iq].Count > 0)
{
MinHeapItem item = m_heaps[iq].RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
}
// To get the fair queing, we cycle through each of the
// queues when finding an element to dequeue.
// We pull (NumberOfQueues - QueueIndex) items from each queue in order
// to give lower numbered queues a higher priority and higher percentage
// of the bandwidth.
// Check for more items to be pulled from the current queue
if (m_heaps[m_nextQueue].Count > 0 && m_countFromQueue > 0)
{
m_countFromQueue--;
MinHeapItem item = m_heaps[m_nextQueue].RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
// Find the next non-immediate queue with updates in it
for (int i = 0; i < NumberOfQueues; ++i)
{
m_nextQueue = (uint)((m_nextQueue + 1) % NumberOfQueues);
m_countFromQueue = m_queueCounts[m_nextQueue];
// if this is one of the immediate queues, just skip it
if (m_nextQueue < NumberOfImmediateQueues)
{
continue;
}
if (m_heaps[m_nextQueue].Count > 0)
{
m_countFromQueue--;
MinHeapItem item = m_heaps[m_nextQueue].RemoveMin();
m_lookupTable.Remove(item.Value.Entity.LocalId);
timeinqueue = Util.EnvironmentTickCountSubtract(item.EntryTime);
value = item.Value;
return(true);
}
}
timeinqueue = 0;
value = default(IEntityUpdate);
return(false);
}
