public JobStatus HandleJobStatus(ObjectId recordid)
            {
                RestAPI api             = new RestAPI();
                string  recordIdStr     = recordid.ToString();
                JObject obj             = api.StatusRequest(recordIdStr);
                bool    messageReceived = obj.GetValue("Successful").Value <bool>();
                int     attempts        = 0;

                while ((!messageReceived) && (attempts < maxRequestAttempts))
                {
                    obj             = api.StatusRequest(recordIdStr);
                    messageReceived = obj.GetValue("Successful").Value <bool>();
                    attempts++;
                }
                Tuple <ServerResponse, JobStatus> response = api.ParseStatusResponse(obj);

                return(response.Item2);
            }
Exemple #2
0
        public ServerResponse ReturnFinishedJob(byte[] zippedBytes)
        {
            ServerResponse response = ServerResponse.ServerError;

            try
            {
                byte[]     recordBytes = RestAPI.Unzip(zippedBytes);
                HtmlRecord record      = BSON.Deserialize <HtmlRecord>(recordBytes);
                CrawlerManager.Instance.RemoveJob(record.recordid);
                DataManager.Instance.UpdateEntry(record);
                response = ServerResponse.Success;
            }
            catch (Exception ex)
            {
                System.Diagnostics.Debug.Print(ex.ToString());
                response = ServerResponse.ServerError;
            }
            return(response);
        }
            public void Send()
            {
                RestAPI api = new RestAPI();

                while (!processDestroyed)
                {
                    HtmlRecord record = messageQueue.Take();
                    foreach (HtmlResults results in record.results.Values)
                    {
                        results.links = null;
                    }
                    byte[] recordString = record.ToBson <HtmlRecord>();
                    try
                    {
                        JObject obj             = api.EnqueueJob(recordString);
                        bool    messageReceived = obj.GetValue("Successful").Value <bool>();
                        if (messageReceived)
                        {
                            ServerResponse response = api.ParseResponse(obj);
                            if (response != ServerResponse.Success)
                            {
                                messageQueue.Add(record);
                            }
                            else
                            {
                                System.Diagnostics.Debug.Print("[" + DateTime.Now.ToString() + "] Sent: " +
                                                               record.domain.AbsoluteUri);
                            }
                        }
                        else
                        {
                            messageQueue.Add(record);
                        }
                    }
                    catch (WebException ex)
                    {
                        System.Diagnostics.Debug.Print(ex.ToString());
                        messageQueue.Add(record);
                    }
                }
            }