Example #1
0
        public async void StartSloader()
        {
            SloaderRunner = new SloaderRunner(SloaderConfigLoader.Parse(this.Input, new Dictionary <string, string>()));
            var result = await SloaderRunner.RunAllCrawlers();

            Output = result.ToJson();
        }
Example #2
0
        public async Task <ActionResult> Index()
        {
            var viewModel = new HomeIndexViewModel();

            viewModel.SloaderConfigPath = ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath];
            if (viewModel.SloaderConfigPathIsConfigured)
            {
                var config = await SloaderConfigLoader.GetAsync(ConfigurationManager.AppSettings[ConfigKeys.SloaderConfigPath], new Dictionary <string, string>());

                viewModel.SloaderConfig = config;

                var secrets = new SloaderSecrets();
                secrets.TwitterConsumerKey    = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerKey];
                secrets.TwitterConsumerSecret = ConfigurationManager.AppSettings[ConfigKeys.SecretTwitterConsumerSecret];

                viewModel.SloaderConfigIsTwitterConsumerConfigured = secrets.IsTwitterConsumerConfigured;
            }

            var azureWebJobStorage = ConfigurationManager.AppSettings[ConfigKeys.AzureWebJobStorage];

            if (azureWebJobStorage != null)
            {
                CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureWebJobStorage);
                CloudBlobClient     blobClient     = storageAccount.CreateCloudBlobClient();
                CloudBlobContainer  container      = blobClient.GetContainerReference(Constants.SloaderAzureBlobContainer);
                container.CreateIfNotExists();

                var file = await container.GetBlobReferenceFromServerAsync(Constants.SloaderAzureBlobFileName);

                using (var memory = new MemoryStream())
                    using (var reader = new StreamReader(memory))
                    {
                        await file.DownloadToStreamAsync(memory);

                        memory.Seek(0, SeekOrigin.Begin);

                        viewModel.ResultText = reader.ReadToEnd();
                    }

                JsonConverter[] converters = { new CrawlerResultConverter() };
                viewModel.ResultData = JsonConvert.DeserializeObject <CrawlerRun>(viewModel.ResultText, converters);
            }

            return(View(viewModel));
        }
        public void Parser_Can_Embed_Secrets()
        {
            StringBuilder fakeYaml = new StringBuilder();

            fakeYaml.AppendLine("Secrets:");
            fakeYaml.AppendLine("  TwitterConsumerKey: $$SecretPlaceholder1$$");
            fakeYaml.AppendLine("  TwitterConsumerSecret: $$SecretPlaceholder2$$");
            fakeYaml.AppendLine("");
            fakeYaml.AppendLine("Crawler:");
            fakeYaml.AppendLine("  TwitterTimelinesToCrawl:");
            fakeYaml.AppendLine("  - Handle: codeinsideblog");

            Dictionary <string, string> secrets = new Dictionary <string, string>();

            secrets.Add("SecretPlaceholder1", "This is the secret key");
            secrets.Add("SecretPlaceholder2", "This is the secret secret");

            var result = SloaderConfigLoader.Parse(fakeYaml.ToString(), secrets);

            Assert.Equal("This is the secret key", result.Secrets.TwitterConsumerKey);
            Assert.Equal("This is the secret secret", result.Secrets.TwitterConsumerSecret);
        }