public async void StartSloader()
{
SloaderRunner = new SloaderRunner(SloaderConfigLoader.Parse(this.Input, new Dictionary <string, string>()));
var result = await SloaderRunner.RunAllCrawlers();
Output = result.ToJson();
}
public async Task <ActionResult> Index()
{
var viewModel = new HomeIndexViewModel();
viewModel.SloaderConfigPath = ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath];
if (viewModel.SloaderConfigPathIsConfigured)
{
var config = await SloaderConfigLoader.GetAsync(ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath], new Dictionary <string, string>());
viewModel.SloaderConfig = config;
var secrets = new SloaderSecrets();
secrets.TwitterConsumerKey = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerKey];
secrets.TwitterConsumerSecret = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerSecret];
viewModel.SloaderConfigIsTwitterConsumerConfigured = secrets.IsTwitterConsumerConfigured;
}
var azureWebJobStorage = ConfigurationManager.AppSettings[ConfigKeys.AzureWebJobStorage];
if (azureWebJobStorage != null)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureWebJobStorage);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(Constants.SloaderAzureBlobContainer);
container.CreateIfNotExists();
var file = await container.GetBlobReferenceFromServerAsync(Constants.SloaderAzureBlobFileName);
using (var memory = new MemoryStream())
using (var reader = new StreamReader(memory))
{
await file.DownloadToStreamAsync(memory);
memory.Seek(0, SeekOrigin.Begin);
viewModel.ResultText = reader.ReadToEnd();
}
JsonConverter[] converters = { new CrawlerResultConverter() };
viewModel.ResultData = JsonConvert.DeserializeObject <CrawlerRun>(viewModel.ResultText, converters);
}
return(View(viewModel));
}
public void Parser_Can_Embed_Secrets()
{
StringBuilder fakeYaml = new StringBuilder();
fakeYaml.AppendLine("Secrets:");
fakeYaml.AppendLine(" TwitterConsumerKey: $$SecretPlaceholder1$$");
fakeYaml.AppendLine(" TwitterConsumerSecret: $$SecretPlaceholder2$$");
fakeYaml.AppendLine("");
fakeYaml.AppendLine("Crawler:");
fakeYaml.AppendLine(" TwitterTimelinesToCrawl:");
fakeYaml.AppendLine(" - Handle: codeinsideblog");
Dictionary <string, string> secrets = new Dictionary <string, string>();
secrets.Add("SecretPlaceholder1", "This is the secret key");
secrets.Add("SecretPlaceholder2", "This is the secret secret");
var result = SloaderConfigLoader.Parse(fakeYaml.ToString(), secrets);
Assert.Equal("This is the secret key", result.Secrets.TwitterConsumerKey);
Assert.Equal("This is the secret secret", result.Secrets.TwitterConsumerSecret);
}
