/// <summary>
/// Checks the status of a dataflow job and defines the URLs to use to download results when the job is completed.
/// </summary>
/// <param name="dataflowJobLocation">The URL to use to check status for a job.</param>
/// <param name="bingMapsKey">The Bing Maps Key for this job. The same key is used to create the job and download results.</param>
/// <returns>
/// A DownloadDetails object that contains the status of the geocode dataflow job (Completed, Pending, Aborted).
/// When the status is set to Completed, DownloadDetails also contains the links to download the results
/// </returns>
private async Task <DownloadDetails> CheckStatus(string dataflowJobLocation, string bingMapsKey)
{
//Build the HTTP Request to get job status
var uriBuilder = new UriBuilder(dataflowJobLocation + @"?key=" + bingMapsKey + "&output=xml&clientApi=SDSToolkit");
var request = (HttpWebRequest)WebRequest.Create(uriBuilder.Uri);
request.Method = "GET";
var statusDetails = new DownloadDetails()
{
JobStatus = "Pending"
};
using (var response = (HttpWebResponse)await request.GetResponseAsync())
{
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception("An HTTP error status code was encountered when checking job status.");
}
using (var receiveStream = response.GetResponseStream())
{
var reader = XmlReader.Create(receiveStream);
while (reader.Read())
{
if (reader.IsStartElement())
{
if (reader.Name.Equals("Status"))
{
//return job status
statusDetails.JobStatus = reader.ReadElementContentAsString();
break;
}
else if (reader.Name.Equals("Link"))
{
//Set the URL location values for retrieving
// successful and failed job results
reader.MoveToFirstAttribute();
if (reader.Value.Equals("output"))
{
reader.MoveToNextAttribute();
if (reader.Value.Equals("succeeded"))
{
reader.MoveToContent();
statusDetails.SucceededUrl = reader.ReadElementContentAsString();
}
else if (reader.Value.Equals("failed"))
{
reader.MoveToContent();
statusDetails.FailedUrl = reader.ReadElementContentAsString();
}
}
}
}
}
}
}
return(statusDetails);
}
private void DownloadManager_OnDownloadStopped(object sender, DownloadDetails e)
{
RunOnUiThread(() =>
{
_btnStartStop.Text = "START";
_btnStartStop.Enabled = true;
_btnResume.Visibility = ViewStates.Visible;
});
}
private void DownloadManager_OnDownloadProgressChanged(object sender, DownloadDetails downloadDetails, long downloadedSize, long totalSize)
{
RunOnUiThread(() =>
{
_lblFileName.Text = downloadDetails.FileName;
_lblProgress.Text = (int)downloadDetails.CurrentProgress.GetProgress() + "%";
_progress.Progress = (int)downloadDetails.CurrentProgress.GetProgress();
_progress.Indeterminate = false;
});
}
private void DownloadManager_OnDownloadCompleted(object sender, DownloadDetails downloadDetails)
{
RunOnUiThread(() =>
{
_lblFileName.Text = downloadDetails.FileName;
_progress.Indeterminate = false;
_btnStartStop.Enabled = true;
_btnStartStop.Text = "START";
_btnResume.Visibility = ViewStates.Gone;
});
}
private void DownloadManager_OnDownloadStarted(object sender, DownloadDetails downloadDetails, long fileSize)
{
RunOnUiThread(() =>
{
_lblFileName.Text = downloadDetails.FileName;
_lblProgress.Text = "";
_btnStartStop.Enabled = true;
_btnResume.Visibility = ViewStates.Gone;
_progress.Indeterminate = false;
_progress.Progress = 0;
_btnStartStop.Text = "STOP";
});
}
/// <summary>
/// Downloads job results to files names Success.txt (successfully geocoded results) and Failed.txt (info about spatial data that was not geocoded successfully).
/// </summary>
/// <param name="statusDetails">Inclues job status and the URLs to use to download all geocoded results.</param>
/// <param name="bingMapsKey">The Bing Maps Key for this job. The same key is used to create the job and get job status. </param>
private async Task <BatchGeocoderResults> DownloadResults(DownloadDetails statusDetails, string bingMapsKey)
{
var results = new BatchGeocoderResults();
//Write the results for data that was geocoded successfully to a file named Success.xml
if (statusDetails.SucceededUrl != null && !statusDetails.SucceededUrl.Equals(String.Empty))
{
//Create a request to download successfully geocoded data. You must add the Bing Maps Key to the
//download location URL provided in the response to the job status request.
var successUriBuilder = new UriBuilder(statusDetails.SucceededUrl + "?clientApi=SDSToolkit&key=" + bingMapsKey);
var successfulRequest = (HttpWebRequest)WebRequest.Create(successUriBuilder.Uri);
successfulRequest.Method = "GET";
using (var response = (HttpWebResponse)await successfulRequest.GetResponseAsync())
{
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception("An HTTP error status code was encountered when downloading results.");
}
using (var receiveStream = response.GetResponseStream())
{
results.Succeeded = await GeocodeFeed.ReadAsync(receiveStream);
}
}
}
//If some spatial data could not be geocoded, write the error information to a file called Failed.xml
if (statusDetails.FailedUrl != null && !statusDetails.FailedUrl.Equals(String.Empty))
{
var failedRequest = (HttpWebRequest)WebRequest.Create(new Uri(statusDetails.FailedUrl + "?clientApi=SDSToolkit&key=" + bingMapsKey));
failedRequest.Method = "GET";
using (var response = (HttpWebResponse)await failedRequest.GetResponseAsync())
{
if (response.StatusCode != HttpStatusCode.OK)
{
throw new Exception("An HTTP error status code was encountered when downloading results.");
}
using (Stream receiveStream = response.GetResponseStream())
{
results.Failed = await GeocodeFeed.ReadAsync(receiveStream);
}
}
}
return(results);
}
private void DownloadManager_OnDownloadErrorOccurred(object sender, DownloadDetails downloadDetails, DownloadManager.Exception errorDetails)
{
RunOnUiThread(() =>
{
_lblFileName.Text = downloadDetails.FileName;
_lblProgress.Text = "0%";
_progress.Progress = 0;
_progress.Indeterminate = false;
_btnStartStop.Enabled = true;
_btnStartStop.Text = "START";
_btnResume.Visibility = ViewStates.Visible;
});
}
/// <summary>
/// Method to geocode a set of data.
/// </summary>
/// <param name="dataFeed">GeocodeFeed which contains the data to batch geocode/reverse geocode.</param>
/// <param name="bingMapsKey">Bing Maps key to use for accessing service.</param>
/// <returns>The results of the batch geocoding process.</returns>
public async Task <BatchGeocoderResults> Geocode(GeocodeFeed dataFeed, string bingMapsKey)
{
BatchGeocoderResults results;
try
{
ReportStatus("Creating batch geocode job.");
string dataflowJobLocation = await CreateJob(dataFeed, bingMapsKey);
ReportStatus("Job created and being processed.");
//Continue to check the dataflow job status until the job has completed
var statusDetails = new DownloadDetails();
do
{
statusDetails = await CheckStatus(dataflowJobLocation, bingMapsKey);
if (statusDetails.JobStatus == "Aborted")
{
ReportStatus("Batch geocode job aborted.");
return(new BatchGeocoderResults()
{
Error = "Batch geocode job was aborted due to an error."
});
}
if (statusDetails.JobStatus.Equals("Pending"))
{
await Task.Delay(_statusUpdateInterval);
}
}while (statusDetails.JobStatus.Equals("Pending"));
ReportStatus("Batch geocode job completed. Downloading results.");
results = await DownloadResults(statusDetails, bingMapsKey);
ReportStatus("Batch geocode results downloaded.");
}
catch (Exception ex)
{
results = new BatchGeocoderResults()
{
Error = ex.Message
};
}
return(results);
}
public DownloadDetails GetDownloadURL()
{
var script = HtmlDocument.DocumentNode.Descendants()
.Where(n => n.Name == "script" && n.InnerText.Contains("dlbutton"))
.First();
var firstPass = script.InnerText.Substring(49);
var partOne = firstPass.Substring(0, 12);
var partTwoStart = firstPass.IndexOf('(') + 1;
var partTwoEnd = firstPass.IndexOf(')');
var partTwoLength = partTwoEnd - partTwoStart;
var secondPass = firstPass.Substring(partTwoEnd);
var partTwo = firstPass.Substring(partTwoStart, partTwoLength);
var computeDT = new DataTable();
var partTwoResult = (int)computeDT.Compute(partTwo, "");
var partThreeStart = secondPass.IndexOf('"') + 1;
var partThreeEnd = secondPass.IndexOf(';') - 1;
var partThreeLength = partThreeEnd - partThreeStart;
var partThree = secondPass.Substring(partThreeStart, partThreeLength);
var URL = partOne + partTwoResult.ToString() + partThree;
var details = new DownloadDetails()
{
URL = URL,
FileName = partThree.Substring(1)
};
return(details);
throw new ScrapeException("Could not scrape conditions.", Html);
}
