/// <summary>
/// Put a new request in the queue. Returns a DataRequestDescriptor matching the request. If there is
/// already a request pending for the same source, the existing descriptor is returned.
/// </summary>
/// <param name="request"></param>
/// <returns></returns>
static public DataRequest Request(DataRequestDescriptor request)
{
// TODO: actually parse the request and differentiate between different DataRequest types
DataRequestHTTP drd = new DataRequestHTTP(request);
// before anything, try to fulfill the request from cache
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataStore: New request for " + request.Source);
#endif
/*
* if (drd.TryCache())
* {
#if VERBOSE
* Log.Write(Log.Levels.Verbose, "DataStore: Request fulfilled from cache, no queue required");
#endif
* if (drd.RequestDescriptor.CompletionCallback != null)
* {
#if VERBOSE
* Log.Write(Log.Levels.Verbose, "DataStore: calling completion callback...");
#endif
* drd.RequestDescriptor.CompletionCallback(drd);
* }
* return drd;
* }
*/
lock (m_lock)
{
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataStore: Request enqueued.");
#endif
m_pendingRequests.Insert(0, drd);
}
return(drd);
}
/// <summary>
/// Asynchronous read callback. Called upon completion of stream reading.
/// </summary>
/// <param name="asyncResult">Result state.</param>
private static void ReadCallback(IAsyncResult asyncResult)
{
DataRequestHTTP dataRequest = asyncResult.AsyncState as DataRequestHTTP;
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataRequest: async read finished for " + dataRequest.m_request.Source);
Log.Write(Log.Levels.Verbose, "DataRequest: completed? is " + asyncResult.IsCompleted);
#endif
int newBytes = dataRequest.m_responseStream.EndRead(asyncResult);
m_totalBytes += newBytes;
dataRequest.m_bytesRead += newBytes;
if (dataRequest.m_bytesRead < dataRequest.m_buffer.Length)
{
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataRequest: not complete, rescheduling for the rest. bytes read so far: " + dataRequest.m_bytesRead);
#endif
dataRequest.m_responseStream.BeginRead(dataRequest.m_buffer, dataRequest.m_bytesRead, dataRequest.m_buffer.Length - dataRequest.m_bytesRead, new AsyncCallback(ReadCallback), dataRequest);
return;
}
dataRequest.m_responseStream.Close();
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataRequest: stream wrote " + dataRequest.m_bytesRead + " bytes");
#endif
dataRequest.m_contentStream = new MemoryStream(dataRequest.m_buffer);
dataRequest.m_state = DataRequestState.Finished;
}
/// <summary>
/// Asynchronous response callback. Called upon completion of the WebRequest.
/// </summary>
/// <param name="asyncResult">Result state.</param>
private static void ResponseCallback(IAsyncResult asyncResult)
{
DataRequestHTTP dataRequest = asyncResult.AsyncState as DataRequestHTTP;
try
{
WebResponse response = dataRequest.m_webRequest.EndGetResponse(asyncResult);
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataRequest: response received for " + dataRequest.m_request.Source);
Log.Write(Log.Levels.Verbose, "DataRequest: Content Length = " + response.ContentLength);
Log.Write(Log.Levels.Verbose, "DataRequest: Content Type = " + response.ContentType);
Log.Write(Log.Levels.Verbose, "DataRequest: starting async read...");
#endif
if (response.ContentLength > 0)
{
dataRequest.m_headers = response.Headers;
dataRequest.m_responseStream = response.GetResponseStream();
dataRequest.m_buffer = new byte[response.ContentLength];
dataRequest.m_bytesRead = 0;
dataRequest.m_readResult = dataRequest.m_responseStream.BeginRead(dataRequest.m_buffer, 0, (int)(response.ContentLength), new AsyncCallback(ReadCallback), dataRequest);
}
else
{
response.Close();
//TODO: hack to get responses with < 0 content length to work, which includes TerraServer layers
using (System.Net.WebClient client = new WebClient())
{
dataRequest.m_buffer = client.DownloadData(response.ResponseUri.OriginalString);
dataRequest.m_bytesRead = dataRequest.m_buffer.Length;
dataRequest.m_contentStream = new MemoryStream(dataRequest.m_buffer);
dataRequest.m_state = DataRequestState.Finished;
}
}
}
catch (System.Net.WebException ex)
{
Log.Write(Log.Levels.Warning, "DataRequest: exception caught trying to access " + dataRequest.Source);
Log.Write(ex);
dataRequest.m_state = DataRequestState.Error;
}
}
/// <summary>
/// Handle pending and active data requests. Call this function regularly from the background worker thread.
/// </summary>
static public void Update()
{
#if VERBOSE
if (PendingRequestCount > 0 || ActiveRequestCount > 0)
{
Log.Write(Log.Levels.Verbose, "DataStore: Update() : " + PendingRequestCount + " requests pending, " + ActiveRequestCount + " active.");
}
#endif
// clean out finished requests.
for (int i = 0; i < m_activeRequests.Count; i++)
{
DataRequestHTTP dr = m_activeRequests[i] as DataRequestHTTP;
if (dr.State != DataRequestState.InProcess)
{
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataStore: removing request " + dr.Source + " in state " + dr.State.ToString());
#endif
if (dr.State == DataRequestState.Finished && dr.RequestDescriptor.CompletionCallback != null)
{
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataStore: calling completion callback...");
#endif
dr.RequestDescriptor.CompletionCallback(dr);
}
else if (dr.State == DataRequestState.Error)
{
dr.State = DataRequestState.Delayed;
dr.NextTry = DateTime.Now + TimeSpan.FromSeconds(120);
Log.Write(Log.Levels.Warning, "DataStore: request " + dr.Source + " has error, delaying until " + dr.NextTry.ToLongTimeString());
m_pendingRequests.Add(dr);
}
m_activeRequests.Remove(dr);
}
}
lock (m_lock)
{
// if we're ready to activate new requests, first sort them according to current priority
if (ActiveRequestCount < m_maxActiveRequests)
{
foreach (DataRequest dr in m_pendingRequests)
{
dr.UpdatePriority();
//if (dr.Priority < 0)
// dr.Cancel();
if (dr.State == DataRequestState.Delayed && dr.NextTry < DateTime.Now)
{
dr.State = DataRequestState.NoCache;
}
}
// also clean up cancelled requests
for (int i = 0; i < PendingRequestCount; i++)
{
DataRequest dr = m_pendingRequests[i] as DataRequest;
if (dr.State == DataRequestState.Queued)
{
if (dr.TryCache())
{
dr.RequestDescriptor.CompletionCallback(dr);
m_pendingRequests.Remove(dr);
}
else
{
dr.State = DataRequestState.NoCache;
}
}
if (dr.State == DataRequestState.Cancelled)
{
m_pendingRequests.Remove(dr);
}
}
m_pendingRequests.Sort();
}
}
// see if we can start any more requests
while ((PendingRequestCount > 0) && (ActiveRequestCount < m_maxActiveRequests))
{
// look for first pending request
DataRequest drd = null;
lock (m_lock)
{
drd = m_pendingRequests[0] as DataRequest;
// if the top priority request is delayed, we won't find anything better.
// so we can break out immediately.
if (drd.State != DataRequestState.NoCache || drd.NextTry > DateTime.Now)
{
break;
}
m_pendingRequests.RemoveAt(0);
}
#if VERBOSE
Log.Write(Log.Levels.Verbose, "DataStore: Activating request for " + drd.Source);
#endif
// if (!drd.TryCache())
{
drd.Start();
m_activeRequests.Add(drd);
}
}
}