public async Task GetAsyncReturnsRobotTextWhenApiReturnsRobotText()
{
const string ExpectedResponseText = "SomeResponseText";
var httpResponse = new HttpResponseMessage
{
StatusCode = HttpStatusCode.OK,
Content = new StringContent(ExpectedResponseText),
};
var fakeHttpRequestSender = A.Fake <IFakeHttpRequestSender>();
A.CallTo(() => fakeHttpRequestSender.Send(A <HttpRequestMessage> .Ignored)).Returns(httpResponse);
var fakeHttpMessageHandler = new FakeHttpMessageHandler(fakeHttpRequestSender);
var httpClient = new HttpClient(fakeHttpMessageHandler)
{
BaseAddress = new Uri("http://SomeDummyCDNUrl")
};
var robotService = new ApplicationRobotService(httpClient);
var model = new ApplicationRobotModel {
BearerToken = "SomeBearerToken"
};
var result = await robotService.GetAsync(model);
Assert.Equal(ExpectedResponseText, result);
httpResponse.Dispose();
httpClient.Dispose();
fakeHttpMessageHandler.Dispose();
}
private static IEnumerable <string> ProcessRobotsLines(ApplicationRobotModel applicationRobotModel, string baseUrl, string[] robotsLines)
{
for (var i = 0; i < robotsLines.Length; i++)
{
if (!string.IsNullOrWhiteSpace(robotsLines[i]))
{
// remove any user-agent and sitemap lines
var lineSegments = robotsLines[i].Split(new[] { ':' }, 2);
var skipLinesWithSegment = new[] { "User-agent", "Sitemap" };
if (lineSegments.Length > 0 && skipLinesWithSegment.Contains(lineSegments[0], StringComparer.OrdinalIgnoreCase))
{
robotsLines[i] = string.Empty;
}
// rewrite the URL to swap any child application address prefix for the composite UI address prefix
var pathRootUri = new Uri(applicationRobotModel.RobotsURL);
var appBaseUrl = $"{pathRootUri.Scheme}://{pathRootUri.Authority}";
if (robotsLines[i].Contains(appBaseUrl, StringComparison.InvariantCultureIgnoreCase))
{
robotsLines[i] = robotsLines[i].Replace(appBaseUrl, baseUrl, StringComparison.InvariantCultureIgnoreCase);
}
}
}
return(robotsLines.Where(w => !string.IsNullOrWhiteSpace(w)));
}
private async Task <Robot> CallHttpClientTxtAsync(ApplicationRobotModel model)
{
using (var request = new HttpRequestMessage(HttpMethod.Get, model.RobotsURL))
{
if (!string.IsNullOrWhiteSpace(model.BearerToken))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", model.BearerToken);
}
request.Headers.Add(HeaderNames.Accept, MediaTypeNames.Text.Plain);
var response = await httpClient.SendAsync(request);
response.EnsureSuccessStatusCode();
var responseString = await response.Content.ReadAsStringAsync();
var result = new Robot();
using (var reader = new StringReader(responseString))
{
result.Append(reader.ReadToEnd());
return(result);
}
}
}
public async Task GetAsyncReturnsExceptionIfNoRobotsTextFound()
{
var robotService = new ApplicationRobotService(defaultHttpClient);
var model = new ApplicationRobotModel {
BearerToken = "SomeBearerToken"
};
var exceptionResult = await Assert.ThrowsAsync <InvalidOperationException>(async() => await robotService.GetAsync(model));
Assert.StartsWith("An invalid request URI was provided.", exceptionResult.Message, StringComparison.OrdinalIgnoreCase);
}
public async Task <string> GetAsync(ApplicationRobotModel model)
{
if (model == null)
{
return(null);
}
var responseTask = await CallHttpClientTxtAsync(model);
return(responseTask?.Data);
}
private static void AppendApplicationRobotData(ApplicationRobotModel applicationRobotModel, string applicationRobotsText, string baseUrl, Robot robot)
{
var robotsLines = applicationRobotsText.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
var robotResults = ProcessRobotsLines(applicationRobotModel, baseUrl, robotsLines);
foreach (var robotResult in robotResults)
{
if (!robot.Lines.Contains(robotResult))
{
robot.Add(robotResult);
}
}
}
private async Task <List <ApplicationRobotModel> > CreateApplicationRobotModelTasksAsync(IEnumerable <AppRegistrationModel> appRegistrationModel)
{
var bearerToken = User.Identity.IsAuthenticated ? await bearerTokenRetriever.GetToken(HttpContext).ConfigureAwait(false) : null;
var applicationRobotModels = new List <ApplicationRobotModel>();
foreach (var path in appRegistrationModel)
{
logger.LogInformation($"{nameof(Action)}: Getting child robots.txt for: {path.Path}");
var applicationRobotModel = new ApplicationRobotModel
{
Path = path.Path,
RobotsURL = path.RobotsURL.ToString(),
BearerToken = bearerToken,
};
applicationRobotModel.RetrievalTask = applicationRobotService.GetAsync(applicationRobotModel);
applicationRobotModels.Add(applicationRobotModel);
}
return(applicationRobotModels);
}
