public async Task Should_be_found()
{
var response = new List <MessagesView>();
var endpointName = Conventions.EndpointNamingConvention(typeof(ReceiverRemote));
//search for the message type
var searchString = typeof(MyMessage).Name;
var context = await Define <MyContext>()
.WithEndpoint <Sender>(b => b.When((bus, c) => bus.Send(new MyMessage())))
.WithEndpoint <ReceiverRemote>()
.Done(async c =>
{
var result = await this.TryGetMany <MessagesView>($"/api/endpoints/{endpointName}/messages/search/" + searchString, instanceName: ServiceControlInstanceName);
response = result;
return(result && response.Count == 1);
})
.Run();
var expectedRemote1InstanceId = InstanceIdGenerator.FromApiUrl(SettingsPerInstance[ServiceControlAuditInstanceName].ApiUrl);
var remote1Message = response.SingleOrDefault(msg => msg.MessageId == context.Remote1MessageId);
Assert.NotNull(remote1Message, "Remote1 message not found");
Assert.AreEqual(expectedRemote1InstanceId, remote1Message.InstanceId, "Remote1 instance id mismatch");
}
public async Task Should_be_found()
{
SetInstanceSettings = ConfigureRemoteInstanceForMasterAsWellAsAuditAndErrorQueues;
var response = new List <MessagesView>();
var endpointName = Conventions.EndpointNamingConvention(typeof(ReceiverRemote));
var context = await Define <MyContext>(Remote1, Master)
.WithEndpoint <Sender>(b => b.When((bus, c) => bus.Send(new MyMessage())))
.WithEndpoint <ReceiverRemote>()
.Done(async c =>
{
var result = await this.TryGetMany <MessagesView>($"/api/endpoints/{endpointName}/messages/", instanceName: Master);
response = result;
return(result && response.Count == 1);
})
.Run();
var expectedRemote1InstanceId = InstanceIdGenerator.FromApiUrl(SettingsPerInstance[Remote1].ApiUrl);
var remote1Message = response.SingleOrDefault(msg => msg.MessageId == context.Remote1MessageId);
Assert.NotNull(remote1Message, "Remote1 message not found");
Assert.AreEqual(expectedRemote1InstanceId, remote1Message.InstanceId, "Remote1 instance id mismatch");
}
public void SetUp()
{
ConfigurationManager.AppSettings["ServiceControl/ForwardAuditMessages"] = bool.FalseString;
ConfigurationManager.AppSettings["ServiceControl/ForwardErrorMessages"] = bool.FalseString;
ConfigurationManager.AppSettings["ServiceControl/AuditRetentionPeriod"] = TimeSpan.FromHours(10).ToString();
ConfigurationManager.AppSettings["ServiceControl/ErrorRetentionPeriod"] = TimeSpan.FromDays(10).ToString();
testApi = new TestApi()
{
Settings = new Settings("TestService")
{
Port = 3333,
RemoteInstances = new[]
{
new RemoteInstanceSetting
{
ApiUri = "http://localhost:33334/api",
QueueAddress = "remote1"
}
}
}
};
localInstanceId = InstanceIdGenerator.FromApiUrl(testApi.Settings.ApiUrl);
remote1InstanceId = InstanceIdGenerator.FromApiUrl(testApi.Settings.RemoteInstances[0].ApiUri);
}
public async Task Should_be_found()
{
var response = new List <MessagesView>();
var context = await Define <MyContext>()
.WithEndpoint <Sender>(b => b.When(async(bus, c) =>
{
await bus.Send(new MyMessage());
await bus.SendLocal(new MyMessage());
}))
.WithEndpoint <Receiver>()
.Done(async c =>
{
var result = await this.TryGetMany <MessagesView>("/api/messages/", instanceName: ServiceControlInstanceName);
response = result;
return(result && response.Count == 2);
})
.Run();
var expectedMasterInstanceId = InstanceIdGenerator.FromApiUrl(SettingsPerInstance[ServiceControlInstanceName].ApiUrl);
var expectedAuditInstanceId = InstanceIdGenerator.FromApiUrl(SettingsPerInstance[ServiceControlAuditInstanceName].ApiUrl);
Assert.AreNotEqual(expectedMasterInstanceId, expectedAuditInstanceId);
var sentMessage = response.SingleOrDefault(msg => msg.MessageId == context.SentMessageId);
Assert.NotNull(sentMessage, "Sent message not found");
Assert.AreEqual(expectedAuditInstanceId, sentMessage.InstanceId, "Audit instance id mismatch");
var sentLocalMessage = response.SingleOrDefault(msg => msg.MessageId == context.SentLocalMessageId);
Assert.NotNull(sentLocalMessage, "Sent local message not found");
Assert.AreEqual(expectedAuditInstanceId, sentLocalMessage.InstanceId, "Audit instance id mismatch");
}
private async Task <HttpResponseMessage> RemoteCall(HttpRequestMessage currentRequest, string instanceId)
{
var remoteUri = InstanceIdGenerator.ToApiUri(instanceId);
var instanceUri = currentRequest.RedirectToRemoteUri(remoteUri);
var httpClient = HttpClientFactory();
try
{
currentRequest.RequestUri = instanceUri;
if (currentRequest.Method == HttpMethod.Get)
{
currentRequest.Content = null;
}
currentRequest.Headers.Host = remoteUri.Authority; //switch the host header to the new instance host
var rawResponse = await httpClient.SendAsync(currentRequest).ConfigureAwait(false);
return(rawResponse);
}
catch (Exception exception)
{
logger.Warn($"Failed to query remote instance at {remoteUri}.", exception);
return(new HttpResponseMessage(HttpStatusCode.InternalServerError));
}
}
public UniqueDataScrapersTests()
{
instanceIdGenerator = new InstanceIdGenerator();
htmlParserProvider = new DefaultHtmlParserProvider();
pageDownloader = new DefaultPageDownloader();
webClientProvider = new DefaultWebClientProvider();
webServices = new DefaultWebServices(htmlParserProvider, webClientProvider, pageDownloader);
uniqueDataProvider = new DefaultUniqueDataProvider(instanceIdGenerator);
scrapingServices = new DefaultUniqueDataScrapingServices(uniqueDataProvider, webServices);
}
public async Task Should_be_forwarded_to_remote()
{
SetInstanceSettings = ConfigureRemoteInstanceForMasterAsWellAsAuditAndErrorQueues;
var context = new MyContext();
HttpResponseMessage response = null;
MessagesView capturedMessage = null;
await Define(context, Remote1, Master)
.WithEndpoint <RemoteEndpoint>(b => b.Given(bus =>
{
context.Remote1InstanceId = InstanceIdGenerator.FromApiUrl(addressOfRemote);
bus.SendLocal(new MyMessage());
}))
.Done(async c =>
{
if (string.IsNullOrWhiteSpace(context.Remote1MessageId))
{
return(false);
}
if (!c.Remote1MessageAudited)
{
var result = await TryGetMany <MessagesView>("/api/messages", msg => msg.MessageId == c.Remote1MessageId, Master);
List <MessagesView> messages = result;
if (!result)
{
return(false);
}
c.Remote1MessageAudited = true;
capturedMessage = messages.Single(msg => msg.MessageId == c.Remote1MessageId);
}
response = await GetRaw($"/api/{capturedMessage.BodyUrl}", Master);
Console.WriteLine($"GetRaw for {c.Remote1MessageId} resulted in {response.StatusCode}");
return(response.StatusCode == HttpStatusCode.OK);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(context.Remote1MessageContentType, response.Content.Headers.ContentType.ToString(), "ContentType mismatch");
Assert.NotNull(response.Content.Headers.Expires, "Expires header missing");
Assert.GreaterOrEqual(response.Content.Headers.Expires.Value, DateTimeOffset.UtcNow.AddDays(360), "Unexpected Expires datetime year value");
Assert.NotNull(response.Content.Headers.ContentLength, "ContentLength not set");
Assert.AreEqual(context.Remote1MessageBody.Length, response.Content.Headers.ContentLength.Value, "ContentLength mismatch");
var body = await response.Content.ReadAsByteArrayAsync();
Assert.AreEqual(context.Remote1MessageBody, body, "Body bytes mismatch");
Assert.NotNull(response.Headers.GetValues("ETag").SingleOrDefault(), "Etag not set");
}
public async Task Should_be_forwarded_to_audit_instance()
{
CustomServiceControlAuditSettings = s => addressOfAuditInstance = s.ApiUrl;
HttpResponseMessage response = null;
MessagesView capturedMessage = null;
var context = await Define <MyContext>()
.WithEndpoint <RemoteEndpoint>(b => b.When(async(bus, ctx) =>
{
ctx.AuditInstanceId = InstanceIdGenerator.FromApiUrl(addressOfAuditInstance);
await bus.SendLocal(new MyMessage());
}))
.Done(async c =>
{
if (string.IsNullOrWhiteSpace(c.AuditInstanceMessageId))
{
return(false);
}
if (!c.MessageAudited)
{
var result = await this.TryGetMany <MessagesView>("/api/messages", msg => msg.MessageId == c.AuditInstanceMessageId, ServiceControlInstanceName);
List <MessagesView> messages = result;
if (!result)
{
return(false);
}
c.MessageAudited = true;
capturedMessage = messages.Single(msg => msg.MessageId == c.AuditInstanceMessageId);
}
response = await this.GetRaw($"/api/{capturedMessage.BodyUrl}", ServiceControlInstanceName);
Console.WriteLine($"GetRaw for {c.AuditInstanceMessageId} resulted in {response.StatusCode}");
return(response.StatusCode == HttpStatusCode.OK);
})
.Run(TimeSpan.FromMinutes(2));
Assert.AreEqual(context.MessageContentType, response.Content.Headers.ContentType.ToString(), "ContentType mismatch");
Assert.NotNull(response.Content.Headers.Expires, "Expires header missing");
Assert.GreaterOrEqual(response.Content.Headers.Expires.Value, DateTimeOffset.UtcNow.AddDays(360), "Unexpected Expires datetime year value");
Assert.NotNull(response.Content.Headers.ContentLength, "ContentLength not set");
Assert.AreEqual(context.MessageBody.Length, response.Content.Headers.ContentLength.Value, "ContentLength mismatch");
var body = await response.Content.ReadAsByteArrayAsync();
Assert.AreEqual(context.MessageBody, body, "Body bytes mismatch");
Assert.NotNull(response.Headers.GetValues("ETag").SingleOrDefault(), "Etag not set");
}
public static DefaultMoveScrapingServices MakeDefaultMoveScrapingServices()
{
var instanceIdGenerator = new InstanceIdGenerator();
var htmlParserProvider = new DefaultHtmlParserProvider();
var moveProvider = new DefaultMoveProvider(instanceIdGenerator);
var pageDownloader = new DefaultPageDownloader();
var webClientProvider = new DefaultWebClientProvider();
var webServices = new DefaultWebServices(htmlParserProvider, webClientProvider, pageDownloader);
return(new DefaultMoveScrapingServices(moveProvider, webServices));
}
public void SetUp()
{
var instanceIdGenerator = new InstanceIdGenerator();
var htmlParserProvider = new DefaultHtmlParserProvider();
var movementProvider = new DefaultMovementProvider(instanceIdGenerator);
var pageDownloader = new DefaultPageDownloader();
var webClientProvider = new DefaultWebClientProvider();
var webServices = new DefaultWebServices(htmlParserProvider, webClientProvider, pageDownloader);
_scrapingServices = new DefaultMovementScrapingServices(movementProvider, webServices);
}
private static DefaultAttributeScrapingServices MakeAttributeScrapingServices()
{
var instanceIdGenerator = new InstanceIdGenerator();
var htmlParserProvider = new DefaultHtmlParserProvider();
var attributeProvider = new DefaultAttributeProvider(instanceIdGenerator);
var pageDownloader = new DefaultPageDownloader();
var webClientProvider = new DefaultWebClientProvider();
var webServices = new DefaultWebServices(htmlParserProvider, webClientProvider, pageDownloader);
var scrapingServices = new DefaultAttributeScrapingServices(attributeProvider, webServices);
return(scrapingServices);
}
public Task <Response> Execute(BaseModule module, TIn input)
{
var currentRequest = module.Request;
var instanceId = GetInstance(currentRequest, input);
var localInstanceId = InstanceIdGenerator.FromApiUrl(Settings.ApiUrl);
if (!string.IsNullOrWhiteSpace(instanceId) && instanceId != localInstanceId)
{
return(RemoteCall(currentRequest, instanceId));
}
return(LocalQuery(currentRequest, input, localInstanceId));
}
public Task <HttpResponseMessage> Execute(ApiController controller, TIn input)
{
var currentRequest = controller.Request;
var instanceId = GetInstance(currentRequest, input);
var localInstanceId = InstanceIdGenerator.FromApiUrl(Settings.ApiUrl);
if (!string.IsNullOrWhiteSpace(instanceId) && instanceId != localInstanceId)
{
return(RemoteCall(currentRequest, instanceId));
}
return(LocalQuery(currentRequest, input, localInstanceId));
}
public void SetUp()
{
var instanceIdGenerator = new InstanceIdGenerator();
_htmlParserProvider = new DefaultHtmlParserProvider();
_movementProvider = new DefaultMovementProvider(instanceIdGenerator);
_moveProvider = new DefaultMoveProvider(instanceIdGenerator);
_pageDownloader = new DefaultPageDownloader();
_webClientProvider = new DefaultWebClientProvider();
_attributeProvider = new DefaultAttributeProvider(instanceIdGenerator);
_imageScrapingProvider = new DefaultImageScrapingProvider();
_imageScrapingService = new DefaultImageScrapingService(_imageScrapingProvider);
_uniqueDataProvider = new DefaultUniqueDataProvider(instanceIdGenerator);
_webServices = new DefaultWebServices(_htmlParserProvider, _webClientProvider, _pageDownloader);
_attributeScrapingServices = new DefaultAttributeScrapingServices(_attributeProvider, _webServices);
_moveScrapingServices = new DefaultMoveScrapingServices(_moveProvider, _webServices);
_movementScrapingServices = new DefaultMovementScrapingServices(_movementProvider, _webServices);
_uniqueDataScrapingServices = new DefaultUniqueDataScrapingServices(_uniqueDataProvider, _webServices);
_groundMoveScraper = new GroundMoveScraper(_moveScrapingServices);
_aerialMoveScraper = new AerialMoveScraper(_moveScrapingServices);
_specialMoveScraper = new SpecialMoveScraper(_moveScrapingServices);
_throwMovesScraper = new ThrowMoveScraper(_moveScrapingServices);
_characterMoveScraper = new DefaultCharacterMoveScraper(new List <IMoveScraper>
{
_groundMoveScraper, _aerialMoveScraper, _specialMoveScraper, _throwMovesScraper
});
var attributeScrapers = new List <IAttributeScraper>
{
new AirSpeedScraper(_attributeScrapingServices),
new AirDodgeScraper(_attributeScrapingServices)
};
_movementScraper = new DefaultMovementScraper(_movementScrapingServices);
_characterDataScrapingServices = new DefaultCharacterDataScrapingServices(_imageScrapingService, _movementScraper,
attributeScrapers, _characterMoveScraper, _uniqueDataScrapingServices, _webServices, instanceIdGenerator);
_characterDataScraper = new DefaultCharacterDataScraper(_characterDataScrapingServices);
}
public async Task <HttpResponseMessage> Execute(ApiController controller, TIn input)
{
var remotes = Settings.RemoteInstances;
var currentRequest = controller.Request;
var instanceId = InstanceIdGenerator.FromApiUrl(Settings.ApiUrl);
var tasks = new List <Task <QueryResult <TOut> > >(remotes.Length + 1)
{
LocalCall(currentRequest, input, instanceId)
};
foreach (var remote in remotes)
{
tasks.Add(RemoteCall(currentRequest, remote.ApiAsUri, InstanceIdGenerator.FromApiUrl(remote.ApiUri)));
}
var response = AggregateResults(currentRequest, await Task.WhenAll(tasks).ConfigureAwait(false));
return(Negotiator.FromQueryResult(currentRequest, response));
}
public async Task <dynamic> Execute(BaseModule module, TIn input)
{
var remotes = Settings.RemoteInstances;
var currentRequest = module.Request;
var instanceId = InstanceIdGenerator.FromApiUrl(Settings.ApiUrl);
var tasks = new List <Task <QueryResult <TOut> > >(remotes.Length + 1)
{
LocalCall(currentRequest, input, instanceId)
};
foreach (var remote in remotes)
{
tasks.Add(RemoteCall(currentRequest, remote.ApiUri, InstanceIdGenerator.FromApiUrl(remote.ApiUri)));
}
var response = AggregateResults(currentRequest, await Task.WhenAll(tasks).ConfigureAwait(false));
var negotiate = module.Negotiate;
return(negotiate.WithQueryResult(response, currentRequest));
}
public async Task Should_be_found()
{
SetInstanceSettings = ConfigureRemoteInstanceForMasterAsWellAsAuditAndErrorQueues;
var context = new MyContext();
List <MessagesView> response = new List <MessagesView>();
//search for the message type
var searchString = typeof(MyMessage).Name;
await Define(context, Remote1, Master)
.WithEndpoint <Sender>(b => b.Given((bus, c) =>
{
bus.Send(new MyMessage());
bus.SendLocal(new MyMessage());
}))
.WithEndpoint <ReceiverRemote>()
.Done(async c =>
{
var result = await TryGetMany <MessagesView>("/api/messages/search/" + searchString, instanceName: Master);
response = result;
return(result && response.Count == 2);
})
.Run(TimeSpan.FromSeconds(40));
var expectedMasterInstanceId = InstanceIdGenerator.FromApiUrl(SettingsPerInstance[Master].ApiUrl);
var expectedRemote1InstanceId = InstanceIdGenerator.FromApiUrl(SettingsPerInstance[Remote1].ApiUrl);
var masterMessage = response.SingleOrDefault(msg => msg.MessageId == context.MasterMessageId);
Assert.NotNull(masterMessage, "Master message not found");
Assert.AreEqual(expectedMasterInstanceId, masterMessage.InstanceId, "Master instance id mismatch");
var remote1Message = response.SingleOrDefault(msg => msg.MessageId == context.Remote1MessageId);
Assert.NotNull(remote1Message, "Remote1 message not found");
Assert.AreEqual(expectedRemote1InstanceId, remote1Message.InstanceId, "Remote1 instance id mismatch");
}
public DefaultCharacterDataScraper MakeCharacterDataScraper()
{
var instanceIdGenerator = new InstanceIdGenerator();
_htmlParserProvider = new DefaultHtmlParserProvider();
_movementProvider = new DefaultMovementProvider(instanceIdGenerator);
_moveProvider = new DefaultMoveProvider(instanceIdGenerator);
_pageDownloader = new DefaultPageDownloader();
_webClientProvider = new DefaultWebClientProvider();
_attributeProvider = new DefaultAttributeProvider(instanceIdGenerator);
_imageScrapingProvider = new DefaultImageScrapingProvider();
_imageScrapingService = new DefaultColorScrapingService(_characterCss); //_imageScrapingProvider);
_uniqueDataProvider = new DefaultUniqueDataProvider(instanceIdGenerator);
_webServices = new DefaultWebServices(_htmlParserProvider, _webClientProvider, _pageDownloader);
_attributeScrapingServices = new DefaultAttributeScrapingServices(_attributeProvider, _webServices);
_moveScrapingServices = new DefaultMoveScrapingServices(_moveProvider, _webServices);
_movementScrapingServices = new DefaultMovementScrapingServices(_movementProvider, _webServices);
_uniqueDataScrapingServices = new DefaultUniqueDataScrapingServices(_uniqueDataProvider, _webServices);
_groundMoveScraper = new GroundMoveScraper(_moveScrapingServices);
_aerialMoveScraper = new AerialMoveScraper(_moveScrapingServices);
_specialMoveScraper = new SpecialMoveScraper(_moveScrapingServices);
_throwMoveScraper = new ThrowMoveScraper(_moveScrapingServices);
_characterMoveScraper = new DefaultCharacterMoveScraper(new List <IMoveScraper> {
_groundMoveScraper, _aerialMoveScraper, _specialMoveScraper, _throwMoveScraper
});
var attributeScrapers = AttributeScrapers.AllWithScrapingServices(_attributeScrapingServices, _urlUnderTest);
_movementScraper = new DefaultMovementScraper(_movementScrapingServices);
_characterDataScrapingServices = new DefaultCharacterDataScrapingServices(_imageScrapingService, _movementScraper,
attributeScrapers, _characterMoveScraper, _uniqueDataScrapingServices, _webServices, instanceIdGenerator);
return(new DefaultCharacterDataScraper(_characterDataScrapingServices));
}
public async Task Should_be_work()
{
// instead of setting up a multiple crazy instances we just use the current instance and rely on it forwarding the instance call to itself
CustomServiceControlSettings = s => { addressOfItself = s.ApiUrl; };
FailedMessage failure;
await Define <MyContext>()
.WithEndpoint <FailureEndpoint>(b => b.When(bus => bus.SendLocal(new MyMessage())).DoNotFailOnErrorMessages())
.Done(async c =>
{
if (!c.RetryIssued)
{
var result = await GetFailedMessage(c, ServiceControlInstanceName, FailedMessageStatus.Unresolved);
failure = result;
if (result)
{
c.RetryIssued = true;
await this.Post <object>($"/api/errors/{failure.UniqueMessageId}/retry?instance_id={InstanceIdGenerator.FromApiUrl(addressOfItself)}", null, null, ServiceControlInstanceName);
}
return(false);
}
return(await GetFailedMessage(c, ServiceControlInstanceName, FailedMessageStatus.Resolved));
})
.Run(TimeSpan.FromMinutes(2));
}
private async Task <Response> RemoteCall(Request currentRequest, string instanceId)
{
var remoteUri = InstanceIdGenerator.ToApiUrl(instanceId);
var instanceUri = currentRequest.RedirectToRemoteUri(remoteUri);
var httpClient = HttpClientFactory();
try
{
var method = new HttpMethod(currentRequest.Method);
var requestMessage = new HttpRequestMessage(method, instanceUri);
var streamContent = new StreamContent(currentRequest.Body);
foreach (var currentRequestHeader in currentRequest.Headers)
{
if (contentHeaders.Contains(currentRequestHeader.Key))
{
streamContent.Headers.Add(currentRequestHeader.Key, currentRequestHeader.Value);
}
else
{
requestMessage.Headers.Add(currentRequestHeader.Key, currentRequestHeader.Value);
}
}
if (method == HttpMethod.Post || method == HttpMethod.Put)
{
requestMessage.Content = streamContent;
}
var rawResponse = await httpClient.SendAsync(requestMessage).ConfigureAwait(false);
var headers = rawResponse.Headers.Union(rawResponse.Content.Headers).ToDictionary(k => k.Key, v => v.Value.FirstOrDefault());
var httpStatusCode = (Nancy.HttpStatusCode)Enum.Parse(typeof(HttpStatusCode), rawResponse.StatusCode.ToString(), ignoreCase: true);
return(new Response
{
Contents = stream =>
{
if (httpStatusCode == Nancy.HttpStatusCode.NotFound)
{
Response.NoBody(stream);
}
else
{
rawResponse.Content.CopyToAsync(stream).GetAwaiter().GetResult();
}
},
Headers = headers,
ContentType = rawResponse.Content.Headers.ContentType.ToString(),
StatusCode = httpStatusCode
});
}
catch (Exception exception)
{
logger.Warn($"Failed to query remote instance at {remoteUri}.", exception);
return(new Response
{
StatusCode = Nancy.HttpStatusCode.InternalServerError,
});
}
}
public async Task Should_be_forwarded_and_resolved_on_remote()
{
SetInstanceSettings = ConfigureRemoteInstanceForMasterAsWellAsAuditAndErrorQueues;
var context = new MyContext();
FailedMessage failure;
await Define(context, Remote1, Master)
.WithEndpoint <FailureEndpoint>(b => b.Given(bus => bus.SendLocal(new MyMessage())))
.Done(async c =>
{
if (!c.RetryIssued)
{
var result = await GetFailedMessage(c, Remote1, FailedMessageStatus.Unresolved);
failure = result;
if (result)
{
c.RetryIssued = true;
await Post <object>($"/api/errors/{failure.UniqueMessageId}/retry?instance_id={InstanceIdGenerator.FromApiUrl(addressOfRemote)}", null, null, Master);
}
return(false);
}
return(await GetFailedMessage(c, Remote1, FailedMessageStatus.Resolved));
})
.Run(TimeSpan.FromMinutes(2));
}
