public async Task <ActionResult> Index()
{
var viewModel = new HomeIndexViewModel();
viewModel.SloaderConfigPath = ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath];
if (viewModel.SloaderConfigPathIsConfigured)
{
var config = await SloaderConfigLoader.GetAsync(ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath], new Dictionary <string, string>());
viewModel.SloaderConfig = config;
var secrets = new SloaderSecrets();
secrets.TwitterConsumerKey = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerKey];
secrets.TwitterConsumerSecret = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerSecret];
viewModel.SloaderConfigIsTwitterConsumerConfigured = secrets.IsTwitterConsumerConfigured;
}
var azureWebJobStorage = ConfigurationManager.AppSettings[ConfigKeys.AzureWebJobStorage];
if (azureWebJobStorage != null)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureWebJobStorage);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(Constants.SloaderAzureBlobContainer);
container.CreateIfNotExists();
var file = await container.GetBlobReferenceFromServerAsync(Constants.SloaderAzureBlobFileName);
using (var memory = new MemoryStream())
using (var reader = new StreamReader(memory))
{
await file.DownloadToStreamAsync(memory);
memory.Seek(0, SeekOrigin.Begin);
viewModel.ResultText = reader.ReadToEnd();
}
JsonConverter[] converters = { new CrawlerResultConverter() };
viewModel.ResultData = JsonConvert.DeserializeObject <CrawlerRun>(viewModel.ResultText, converters);
}
return(View(viewModel));
}
public static async Task <CrawlerRun> InvokeCrawler()
{
#if DEBUG
string debugYamlLocation =
"https://raw.githubusercontent.com/Code-Inside/Sloader/master/src/Sloader.Web/App_Data/Sloader.yml";
var config = await SloaderConfig.Load(debugYamlLocation, new Dictionary <string, string>());
#else
var config =
await
SloaderConfig.Load(ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath], new Dictionary <string, string>());
#endif
var secrets = new SloaderSecrets();
secrets.TwitterConsumerKey = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerKey];
secrets.TwitterConsumerSecret = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerSecret];
var crawler = new SloaderRunner(config);
return(await crawler.RunAllCrawlers());
}
