示例#1
0
        /// <summary>
        /// Put a new request in the queue. Returns a DataRequestDescriptor matching the request. If there is
        /// already a request pending for the same source, the existing descriptor is returned.
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        static public DataRequest Request(DataRequestDescriptor request)
        {
            // TODO: actually parse the request and differentiate between different DataRequest types
            DataRequestHTTP drd = new DataRequestHTTP(request);

            // before anything, try to fulfill the request from cache
#if VERBOSE
            Log.Write(Log.Levels.Verbose, "DataStore: New request for " + request.Source);
#endif

/*
 *          if (drd.TryCache())
 *          {
 #if VERBOSE
 *              Log.Write(Log.Levels.Verbose, "DataStore: Request fulfilled from cache, no queue required");
 #endif
 *              if (drd.RequestDescriptor.CompletionCallback != null)
 *              {
 #if VERBOSE
 *                  Log.Write(Log.Levels.Verbose, "DataStore: calling completion callback...");
 #endif
 *                  drd.RequestDescriptor.CompletionCallback(drd);
 *              }
 *              return drd;
 *          }
 */
            lock (m_lock)
            {
#if VERBOSE
                Log.Write(Log.Levels.Verbose, "DataStore: Request enqueued.");
#endif
                m_pendingRequests.Insert(0, drd);
            }
            return(drd);
        }
示例#2
0
        /// <summary>
        /// Asynchronous read callback. Called upon completion of stream reading.
        /// </summary>
        /// <param name="asyncResult">Result state.</param>
        private static void ReadCallback(IAsyncResult asyncResult)
        {
            DataRequestHTTP dataRequest = asyncResult.AsyncState as DataRequestHTTP;

#if VERBOSE
            Log.Write(Log.Levels.Verbose, "DataRequest: async read finished for " + dataRequest.m_request.Source);
            Log.Write(Log.Levels.Verbose, "DataRequest: completed? is " + asyncResult.IsCompleted);
#endif
            int newBytes = dataRequest.m_responseStream.EndRead(asyncResult);
            m_totalBytes            += newBytes;
            dataRequest.m_bytesRead += newBytes;
            if (dataRequest.m_bytesRead < dataRequest.m_buffer.Length)
            {
#if VERBOSE
                Log.Write(Log.Levels.Verbose, "DataRequest: not complete, rescheduling for the rest. bytes read so far: " + dataRequest.m_bytesRead);
#endif
                dataRequest.m_responseStream.BeginRead(dataRequest.m_buffer, dataRequest.m_bytesRead, dataRequest.m_buffer.Length - dataRequest.m_bytesRead, new AsyncCallback(ReadCallback), dataRequest);
                return;
            }
            dataRequest.m_responseStream.Close();
#if VERBOSE
            Log.Write(Log.Levels.Verbose, "DataRequest: stream wrote " + dataRequest.m_bytesRead + " bytes");
#endif

            dataRequest.m_contentStream = new MemoryStream(dataRequest.m_buffer);

            dataRequest.m_state = DataRequestState.Finished;
        }
示例#3
0
        /// <summary>
        /// Asynchronous response callback. Called upon completion of the WebRequest.
        /// </summary>
        /// <param name="asyncResult">Result state.</param>
        private static void ResponseCallback(IAsyncResult asyncResult)
        {
            DataRequestHTTP dataRequest = asyncResult.AsyncState as DataRequestHTTP;

            try
            {
                WebResponse response = dataRequest.m_webRequest.EndGetResponse(asyncResult);

#if VERBOSE
                Log.Write(Log.Levels.Verbose, "DataRequest: response received for " + dataRequest.m_request.Source);
                Log.Write(Log.Levels.Verbose, "DataRequest: Content Length = " + response.ContentLength);
                Log.Write(Log.Levels.Verbose, "DataRequest: Content Type = " + response.ContentType);
                Log.Write(Log.Levels.Verbose, "DataRequest: starting async read...");
#endif
                if (response.ContentLength > 0)
                {
                    dataRequest.m_headers        = response.Headers;
                    dataRequest.m_responseStream = response.GetResponseStream();
                    dataRequest.m_buffer         = new byte[response.ContentLength];
                    dataRequest.m_bytesRead      = 0;
                    dataRequest.m_readResult     = dataRequest.m_responseStream.BeginRead(dataRequest.m_buffer, 0, (int)(response.ContentLength), new AsyncCallback(ReadCallback), dataRequest);
                }
                else
                {
                    response.Close();
                    //TODO: hack to get responses with < 0 content length to work, which includes TerraServer layers
                    using (System.Net.WebClient client = new WebClient())
                    {
                        dataRequest.m_buffer        = client.DownloadData(response.ResponseUri.OriginalString);
                        dataRequest.m_bytesRead     = dataRequest.m_buffer.Length;
                        dataRequest.m_contentStream = new MemoryStream(dataRequest.m_buffer);
                        dataRequest.m_state         = DataRequestState.Finished;
                    }
                }
            }
            catch (System.Net.WebException ex)
            {
                Log.Write(Log.Levels.Warning, "DataRequest: exception caught trying to access " + dataRequest.Source);
                Log.Write(ex);
                dataRequest.m_state = DataRequestState.Error;
            }
        }
示例#4
0
        /// <summary>
        /// Handle pending and active data requests. Call this function regularly from the background worker thread.
        /// </summary>
        static public void Update()
        {
#if VERBOSE
            if (PendingRequestCount > 0 || ActiveRequestCount > 0)
            {
                Log.Write(Log.Levels.Verbose, "DataStore: Update() : " + PendingRequestCount + " requests pending, " + ActiveRequestCount + " active.");
            }
#endif
            // clean out finished requests.
            for (int i = 0; i < m_activeRequests.Count; i++)
            {
                DataRequestHTTP dr = m_activeRequests[i] as DataRequestHTTP;

                if (dr.State != DataRequestState.InProcess)
                {
#if VERBOSE
                    Log.Write(Log.Levels.Verbose, "DataStore: removing request " + dr.Source + " in state " + dr.State.ToString());
#endif
                    if (dr.State == DataRequestState.Finished && dr.RequestDescriptor.CompletionCallback != null)
                    {
#if VERBOSE
                        Log.Write(Log.Levels.Verbose, "DataStore: calling completion callback...");
#endif
                        dr.RequestDescriptor.CompletionCallback(dr);
                    }
                    else if (dr.State == DataRequestState.Error)
                    {
                        dr.State   = DataRequestState.Delayed;
                        dr.NextTry = DateTime.Now + TimeSpan.FromSeconds(120);
                        Log.Write(Log.Levels.Warning, "DataStore: request " + dr.Source + " has error, delaying until " + dr.NextTry.ToLongTimeString());
                        m_pendingRequests.Add(dr);
                    }
                    m_activeRequests.Remove(dr);
                }
            }

            lock (m_lock)
            {
                // if we're ready to activate new requests, first sort them according to current priority
                if (ActiveRequestCount < m_maxActiveRequests)
                {
                    foreach (DataRequest dr in m_pendingRequests)
                    {
                        dr.UpdatePriority();
                        //if (dr.Priority < 0)
                        //    dr.Cancel();

                        if (dr.State == DataRequestState.Delayed && dr.NextTry < DateTime.Now)
                        {
                            dr.State = DataRequestState.NoCache;
                        }
                    }

                    // also clean up cancelled requests
                    for (int i = 0; i < PendingRequestCount; i++)
                    {
                        DataRequest dr = m_pendingRequests[i] as DataRequest;

                        if (dr.State == DataRequestState.Queued)
                        {
                            if (dr.TryCache())
                            {
                                dr.RequestDescriptor.CompletionCallback(dr);
                                m_pendingRequests.Remove(dr);
                            }
                            else
                            {
                                dr.State = DataRequestState.NoCache;
                            }
                        }
                        if (dr.State == DataRequestState.Cancelled)
                        {
                            m_pendingRequests.Remove(dr);
                        }
                    }

                    m_pendingRequests.Sort();
                }
            }

            // see if we can start any more requests
            while ((PendingRequestCount > 0) && (ActiveRequestCount < m_maxActiveRequests))
            {
                // look for first pending request

                DataRequest drd = null;
                lock (m_lock)
                {
                    drd = m_pendingRequests[0] as DataRequest;

                    // if the top priority request is delayed, we won't find anything better.
                    // so we can break out immediately.
                    if (drd.State != DataRequestState.NoCache || drd.NextTry > DateTime.Now)
                    {
                        break;
                    }

                    m_pendingRequests.RemoveAt(0);
                }

#if VERBOSE
                Log.Write(Log.Levels.Verbose, "DataStore: Activating request for " + drd.Source);
#endif
//                if (!drd.TryCache())
                {
                    drd.Start();

                    m_activeRequests.Add(drd);
                }
            }
        }