public async Task TestHttpTwoClientHead()
{
MacroscopeHttpTwoClient Client = new MacroscopeHttpTwoClient();
List <Uri> UrlList = new List <Uri>();
UrlList.Add(new Uri("https://nazuke.github.io/"));
UrlList.Add(new Uri("https://nazuke.github.io/robots.txt"));
foreach (Uri Url in UrlList)
{
this.DebugMsg(string.Format("Url: {0}", Url));
MacroscopeHttpTwoClientResponse ClientResponse = await Client.Head(
Url : Url,
PreProcessCustomRequestHeadersCallback : this.PreProcessHeadRequestHeadersCallback,
PostProcessRequestHttpHeadersCallback : this.PostProcessRequestHttpHeadersCallback
);
HttpResponseMessage Response = ClientResponse.GetResponse();
this.DebugMsg(string.Format("Response.Version: {0}", Response.Version));
Assert.AreEqual(200, (int)Response.StatusCode);
}
}
/**************************************************************************/
public static async Task <string> GetMimeTypeOfUrl(MacroscopeJobMaster JobMaster, Uri TargetUri)
{
MacroscopeHttpTwoClient Client = JobMaster.GetHttpClient();
MacroscopeHttpTwoClientResponse Response = null;
string MimeType = null;
try
{
Response = await Client.Head(TargetUri, ConfigureHeadRequestHeadersCallback, PostProcessRequestHttpHeadersCallback);
if (Response != null)
{
MimeType = Response.GetMimeType().ToString();
}
}
catch (MacroscopeDocumentException ex)
{
DebugMsgStatic(string.Format("MacroscopeDocumentException: {0}", ex.Message));
DebugMsgStatic(string.Format("MacroscopeDocumentException: {0}", TargetUri.ToString()));
}
catch (Exception ex)
{
DebugMsgStatic(string.Format("Exception: {0}", ex.Message));
DebugMsgStatic(string.Format("Exception: {0}", TargetUri.ToString()));
}
return(MimeType);
}
/** -------------------------------------------------------------------- **/
private async Task <bool> _ExecuteHeadCheck()
{
bool IsAvailableCheck = false;
MacroscopeHttpTwoClient Client = this.MsJobMaster.GetHttpClient();
MacroscopeHttpTwoClientResponse ClientResponse = null;
Uri DocUri = null;
try
{
DocUri = new Uri(this.Url);
ClientResponse = await Client.Head(
DocUri,
this.ConfigureHeadRequestHeadersCallback,
this.PostProcessRequestHttpHeadersCallback
);
}
catch (MacroscopeDocumentException ex)
{
this.DebugMsg(string.Format("_ExecuteHeadCheck :: MacroscopeDocumentException: {0}", ex.Message));
}
catch (Exception ex)
{
this.DebugMsg(string.Format("_ExecuteHeadCheck :: Exception: {0}", ex.Message));
}
if (ClientResponse != null)
{
try
{
this.DebugMsg(string.Format("StatusCode: {0}", ClientResponse.GetResponse().StatusCode));
if (ClientResponse.GetResponse() != null)
{
if (ClientResponse.GetResponse().StatusCode == HttpStatusCode.OK)
{
IsAvailableCheck = true;
}
}
else
{
throw new MacroscopeDocumentException("Bad Response in _ExecuteHeadCheck");
}
}
catch (Exception ex)
{
this.DebugMsg(string.Format("_ExecuteHeadCheck :: Exception: {0}", ex.Message));
}
this.ProcessResponseHttpHeaders(Response: ClientResponse);
}
return(IsAvailableCheck);
}
/** -------------------------------------------------------------------- **/
private async Task _ProcessImagePage()
{
MacroscopeHttpTwoClient Client = this.DocCollection.GetJobMaster().GetHttpClient();
MacroscopeHttpTwoClientResponse Response = null;
string ResponseErrorCondition = null;
try
{
Response = await Client.Head(
this.GetUri(),
this.ConfigureImagePageRequestHeadersCallback,
this.PostProcessRequestHttpHeadersCallback
);
}
catch (MacroscopeDocumentException ex)
{
this.DebugMsg(string.Format("_ProcessImagePage :: MacroscopeDocumentException: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
this.SetStatusCode(HttpStatusCode.BadRequest);
this.AddRemark("_ProcessImagePage", ex.Message);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("_ProcessImagePage :: Exception: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
this.SetStatusCode(HttpStatusCode.BadRequest);
this.AddRemark("_ProcessImagePage", ex.Message);
}
if (Response != null)
{
this.ProcessResponseHttpHeaders(Response: Response);
/** Title ---------------------------------------------------------- **/
{
MatchCollection reMatches = Regex.Matches(this.DocUrl, "/([^/]+)$");
string DocumentTitle = null;
foreach (Match match in reMatches)
{
if (match.Groups[1].Value.Length > 0)
{
DocumentTitle = match.Groups[1].Value.ToString();
break;
}
}
if (DocumentTitle != null)
{
this.SetTitle(DocumentTitle, MacroscopeConstants.TextProcessingMode.NO_PROCESSING);
this.DebugMsg(string.Format("TITLE: {0}", this.GetTitle()));
}
else
{
this.DebugMsg(string.Format("TITLE: {0}", "MISSING"));
}
}
/** QR Codes ------------------------------------------------------- **/
if (MacroscopePreferencesManager.GetDetectQrCodeInImage())
{
MacroscopeHttpImageLoader ImageLoader = new MacroscopeHttpImageLoader();
Uri QrCodeImageUri = null;
string QrCodeImageFilename = await ImageLoader.DownloadImageFromUriToFile(JobMaster : this.DocCollection.GetJobMaster(), TargetUri : this.GetUri());
if ((!string.IsNullOrEmpty(QrCodeImageFilename)) && File.Exists(QrCodeImageFilename))
{
MacroscopeQrCodeAnalysis QrCodeAnalysis = new MacroscopeQrCodeAnalysis();
string ResultText = QrCodeAnalysis.Decode(ImageFilename: QrCodeImageFilename);
if (!string.IsNullOrEmpty(ResultText))
{
try
{
QrCodeImageUri = new Uri(ResultText);
}
catch (UriFormatException ex)
{
this.DebugMsg(string.Format("UriFormatException: {0}", ResultText));
this.DebugMsg(string.Format("UriFormatException: {0}", ex.Message));
}
if (QrCodeImageUri != null)
{
MacroscopeLink Outlink = null;
Outlink = this.AddDocumentOutlink(
AbsoluteUrl: QrCodeImageUri.AbsoluteUri,
LinkType: MacroscopeConstants.InOutLinkType.QRCODE,
Follow: true
);
if (Outlink != null)
{
Outlink.SetRawTargetUrl(TargetUrl: QrCodeImageUri.AbsoluteUri);
this.AddRemark("QRCODEIMAGE", "This image appears to be a QR Code.");
}
}
}
}
}
/** ---------------------------------------------------------------- **/
}
if (ResponseErrorCondition != null)
{
this.ErrorCondition = ResponseErrorCondition;
}
}
/** -------------------------------------------------------------------- **/
private async Task _ExecuteHeadRequest()
{
MacroscopeHttpTwoClient Client = this.DocCollection.GetJobMaster().GetHttpClient();
MacroscopeHttpTwoClientResponse ClientResponse = null;
string ResponseErrorCondition = null;
this.SetProcessInlinks();
this.SetProcessHyperlinksIn();
try
{
ClientResponse = await Client.Head(
this.GetUri(),
this.ConfigureHeadRequestHeadersCallback,
this.PostProcessRequestHttpHeadersCallback
);
this.CrawledDate = DateTime.UtcNow;
}
catch (MacroscopeDocumentException ex)
{
this.DebugMsg(string.Format("_ExecuteHeadRequest :: MacroscopeDocumentException: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
this.SetStatusCode(HttpStatusCode.BadRequest);
this.AddRemark("_ExecuteHeadRequest", ex.Message);
}
catch (Exception ex)
{
this.DebugMsg(string.Format("_ExecuteHeadRequest :: Exception: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
this.SetStatusCode(HttpStatusCode.BadRequest);
this.AddRemark("_ExecuteHeadRequest", ex.Message);
}
if (ClientResponse != null)
{
try
{
this.DebugMsg(string.Format("StatusCode: {0}", ClientResponse.GetResponse().StatusCode));
if (ClientResponse.GetResponse() != null)
{
this.SetErrorCondition(ClientResponse.GetResponse().ReasonPhrase);
}
else
{
throw new MacroscopeDocumentException("Bad Response in ExecuteHeadRequest");
}
this.ProcessResponseHttpHeaders(Response: ClientResponse);
if (this.GetIsRedirect())
{
string Location = this.GetUrlRedirectTo();
if (!string.IsNullOrEmpty(Location))
{
MacroscopeLink OutLink = null;
this.SetUrlRedirectTo(Url: Location);
OutLink = this.AddDocumentOutlink(
AbsoluteUrl: Location,
LinkType: MacroscopeConstants.InOutLinkType.REDIRECT,
Follow: true
);
OutLink.SetRawTargetUrl(TargetUrl: this.GetUrlRedirectToRaw());
}
}
}
catch (Exception ex)
{
this.DebugMsg(string.Format("_ExecuteHeadRequest :: Exception: {0}", ex.Message));
ResponseErrorCondition = ex.Message;
}
}
if (ResponseErrorCondition != null)
{
this.ProcessErrorCondition(ResponseErrorCondition);
}
return;
}
