private static async Task <WikiImage> GetImage(int maxRetry = 3) { int tryCount = 0; WikimediaHelper objWikiClient = new WikimediaHelper(); WikiImage objImage = null; do { tryCount++; objImage = await objWikiClient.GetImage(); } while (objImage == null && tryCount < maxRetry); _logger.Info("Got image from wiki after " + tryCount + " retries."); return(objImage); }
static async Task <int> Main(string[] args) { // load environment variables from local file if (File.Exists(Path.Combine(Directory.GetCurrentDirectory(), ".env"))) { DotNetEnv.Env.Load(); } using (SentrySdk.Init()) { var logRepository = log4net.LogManager.GetRepository(Assembly.GetEntryAssembly()); XmlConfigurator.Configure(logRepository, new FileInfo("log4net.config")); GetLogger(); #region Initialize stuff bool msApiEnabled = Convert.ToBoolean(Environment.GetEnvironmentVariable(MS_API_ENABLED)); string msApiKey = Environment.GetEnvironmentVariable(MS_API_KEY_NAME); string msApiEndpoint = Environment.GetEnvironmentVariable(MS_API_ENDPOINT_NAME); bool googleApiEnabled = Convert.ToBoolean(Environment.GetEnvironmentVariable(GOOGLE_API_ENABLED)); string googleAppCredsPath = Environment.GetEnvironmentVariable(GOOGLE_APPLICATION_CREDENTIALS); bool awsApiEnabled = Convert.ToBoolean(Environment.GetEnvironmentVariable(AWS_API_ENABLED)); string awsAccessKey = Environment.GetEnvironmentVariable(AWS_ACCESS_KEY_NAME); string awsSecretKey = Environment.GetEnvironmentVariable(AWS_SECRET_KEY_NAME); string twitterConsumerKey = Environment.GetEnvironmentVariable(TWITTER_CONSUMER_KEY); string twitterConsumerSecretKey = Environment.GetEnvironmentVariable(TWITTER_CONSUMER_SECRET); string twitterAccessKey = Environment.GetEnvironmentVariable(TWITTER_ACCESS_TOKEN); string twitterAccessSecretKey = Environment.GetEnvironmentVariable(TWITTER_ACCESS_TOKEN_SECRET); string healthCheckUrl = Environment.GetEnvironmentVariable(HEALTH_CHECK_URL); if (string.IsNullOrEmpty(healthCheckUrl)) { _logger.Warn("Healthcheck is not configured."); } else { await PingHealthCheck(healthCheckUrl, true); } Stopwatch timer; long timeSpentWikiGetImage; long timeSpentMSDescribeImage; long timeSpentGoogleLabelImage; long timeSpentAWSLabelImage; Censorboard objCensor = new Censorboard(); AzureHelper objAzureClient = null; GoogleHelper objGoogleClient = null; AWSHelper objAWSClient = null; TwitterHelper objTwitter = null; string msSaysWhat = string.Empty; List <string> googleSaysWhat = new List <string>(); List <string> awsSaysWhat = new List <string>(); string imgUrl = string.Empty; byte[] imgBytes = null; if (msApiEnabled) { if (string.IsNullOrEmpty(msApiKey) || string.IsNullOrEmpty(msApiEndpoint)) { _logger.Warn("Azure client not configured properly."); } else { objAzureClient = new AzureHelper(msApiKey, msApiEndpoint); } } if (googleApiEnabled) { if (string.IsNullOrEmpty(googleAppCredsPath) || !File.Exists(googleAppCredsPath)) { _logger.Warn("Google client not configured properly."); } else { objGoogleClient = new GoogleHelper(); } } if (awsApiEnabled) { if (string.IsNullOrEmpty(awsAccessKey) || string.IsNullOrEmpty(awsSecretKey)) { _logger.Warn("AWS client not configured properly."); } else { objAWSClient = new AWSHelper(awsAccessKey, awsSecretKey); } } if (string.IsNullOrEmpty(twitterConsumerKey) || string.IsNullOrEmpty(twitterConsumerSecretKey) || string.IsNullOrEmpty(twitterAccessKey) || string.IsNullOrEmpty(twitterAccessSecretKey)) { _logger.Warn("Twitter client not configured properly"); return(1); } objTwitter = new TwitterHelper(twitterConsumerKey, twitterConsumerSecretKey, twitterAccessKey, twitterAccessSecretKey); #endregion #region Get Image from Wikimedia timer = Stopwatch.StartNew(); WikiImage objImage = await GetImage(3); timer.Stop(); timeSpentWikiGetImage = timer.ElapsedMilliseconds; if (objImage == null) { _logger.Info("No image available even after 3 attempts."); return(2); } _logger.InfoFormat("Got image from Wiki in {1}ms: {0}", objImage.Url, timeSpentWikiGetImage); //#TODO: create entry in database #endregion #region Download the image imgUrl = objImage.Url; if (objImage.Size > 3000000 || objImage.Width > 8192 || objImage.Height > 8192) { imgUrl = objImage.ThumbUrl; } imgBytes = Utility.DownloadImage(imgUrl).Result; if (imgBytes == null || imgBytes.Length == 0) { _logger.Info("No length image. Return"); return(3); } #endregion #region What did the MS say if (objAzureClient == null) { _logger.Info("MS API not enabled. Skipped."); } else { timer = Stopwatch.StartNew(); msSaysWhat = objAzureClient.DescribeImage(imgBytes).Result; timer.Stop(); timeSpentMSDescribeImage = timer.ElapsedMilliseconds; if (msSaysWhat == null) { } else { _logger.InfoFormat("MS said in {1}ms: {0}", msSaysWhat, timeSpentMSDescribeImage); //#TODO: create entry in database } } #endregion #region What did the Google say if (objGoogleClient == null) { _logger.Info("Google API not enabled. Skipped."); googleSaysWhat = new List <string> { "<Skipped>" }; } else { timer = Stopwatch.StartNew(); googleSaysWhat = objGoogleClient.LabelImage(imgBytes).Result; timer.Stop(); timeSpentGoogleLabelImage = timer.ElapsedMilliseconds; _logger.InfoFormat("Google said in {1}ms: {0}", string.Join(',', googleSaysWhat), timeSpentGoogleLabelImage); //#TODO: create entry in database } #endregion #region What did the AWS say if (objAWSClient == null) { _logger.Info("AWS API not enabled. Skipped."); } else { timer = Stopwatch.StartNew(); awsSaysWhat = objAWSClient.LabelImage(imgBytes).Result; timer.Stop(); timeSpentAWSLabelImage = timer.ElapsedMilliseconds; if (awsSaysWhat == null || awsSaysWhat.Count == 0) { } else { _logger.InfoFormat("AWS said in {1}ms: {0}", string.Join(',', awsSaysWhat), timeSpentAWSLabelImage); //#TODO: create entry in database } } #endregion #region Post to twitter string tweetUrl = objTwitter.PostTweet(objImage.DescriptionUrl, imgBytes, msSaysWhat, string.Join(", ", googleSaysWhat), string.Join(", ", awsSaysWhat)); //#TODO: update entry in database _logger.InfoFormat("Tweet posted: {0}", tweetUrl); #endregion _logger.Shutdown(); if (!string.IsNullOrEmpty(healthCheckUrl)) { await PingHealthCheck(healthCheckUrl, false); } return(0); } }
private object CreateImageModel(WikiImage image) { return new { url = Url.Action("Thumbnail", new { id = image.Id }), id = image.Id, filename = image.Filename }; }
private bool IsValidImage(WikimediaResponse objWImage) { WikiImage objIInfo = objWImage.ImageInfo[0]; // check that the file is actually a picture if (objIInfo.MediaType != "BITMAP") { return(false); } // Make sure the picture is big enough if (objIInfo.Width <= 50 || objIInfo.Height <= 50) { return(false); } // Make sure the format is supported string thisExtn = Path.GetExtension(objIInfo.Url); if (Array.IndexOf(FORMATS, thisExtn) < 0) { return(false); } // We got a picture, now let's verify we can use it. //Check file name for bad words if (objCensor.IsBlacklisted(objWImage.Title)) { _logger.InfoFormat("Image discarded, {0}. badword in page title: {1}", objIInfo.DescriptionUrl, objWImage.Title); return(false); } // Check picture title for bad words if (objCensor.IsBlacklisted(objIInfo.Metadata.ObjectName.value.ToString())) { _logger.InfoFormat("Image discarded, {0}. badword in picture title: {1}", objIInfo.DescriptionUrl, objIInfo.Metadata.ObjectName.value.ToString()); return(false); } // Check restrictions for more bad words if (objCensor.IsBlacklisted(objIInfo.Metadata.Restrictions.value.ToString())) { _logger.InfoFormat("Image discarded, {0}. badword in restrictions: {1}", objIInfo.DescriptionUrl, objIInfo.Metadata.Restrictions.value.ToString()); return(false); } // Check file description for bad words if (objIInfo.Metadata.ImageDescription != null) { string cleanedDescription = Utility.GetTextFromHtml(objIInfo.Metadata.ImageDescription.value.ToString()); if (objCensor.IsBlacklisted(cleanedDescription)) { _logger.InfoFormat("Image discarded, {0}. badword in image description: {1}", objIInfo.DescriptionUrl, cleanedDescription); return(false); } if (objCensor.ShouldFilterForPhrase(cleanedDescription)) { _logger.InfoFormat("Image discarded, {0}. blacklisted phrase in image description: {1}", objIInfo.DescriptionUrl, cleanedDescription); return(false); } } // The mediawiki API is awful, there's another list of categories which // is not the same as the one requested by asking for "categories". // Fortunately it's still in the API response, under extmetadata. List <string> lstCateg = new List <string>(); lstCateg.Add(objIInfo.Metadata.Categories.value.ToString()); foreach (dynamic objCateg in objWImage.Category) { lstCateg.Add(objCateg.title.ToString()); } if (objCensor.ShouldFilterForCategory(lstCateg)) { _logger.InfoFormat("Image discarded, {0}. blacklisted phrase in categories", objIInfo.DescriptionUrl); return(false); } // #TODO:: check parent categories for each category in metadata, // and compare them against the blacklist too. This will require // extra API calls // if the picture is used in any wikipage with unwanted themes, we probably // don't want to use it. foreach (dynamic wikipage in objWImage.GlobalUsage) { if (objCensor.IsBlacklisted(wikipage.title.ToString())) { _logger.InfoFormat("Image discarded, {0}. page usage {1}", objIInfo.DescriptionUrl, wikipage.title.ToString()); return(false); } if (objCensor.ShouldFilterForCategory(wikipage.title.ToString())) { _logger.InfoFormat("Image discarded, {0}. page usage {1}", objIInfo.DescriptionUrl, wikipage.title.ToString()); return(false); } } return(true); }