public async Task GetFileContentsTest() { var s3Connector = new S3Connector(region, bucketName); ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(); ICollection <JobPost> jobPostsObjs = await s3Connector.GetFileContents(jobPostsKeys); Assert.True(jobPostsObjs.Count == 2); var javaJobPost = new JobPost("Senior Java Developer", "Start as soon as possible"); javaJobPost.Keywords.Add("java"); javaJobPost.Keywords.Add("maven"); javaJobPost.Keywords.Add("junit"); var cSharpJobPost = new JobPost("Senior Software Developer", "Contact us as soon as possible"); cSharpJobPost.Keywords.Add("c#"); cSharpJobPost.Keywords.Add("dotnet"); cSharpJobPost.Keywords.Add("full stack"); Assert.Equal(javaJobPost, jobPostsObjs.First(jobPost => jobPost.Header == javaJobPost.Header)); Assert.Equal(cSharpJobPost, jobPostsObjs.First(jobPost => jobPost.Header == cSharpJobPost.Header)); }
public async Task GetNoFilesTest() { var s3Connector = new S3Connector(region, bucketName); ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(to : new DateTime(2020, 05, 08)); Assert.True(jobPostsKeys.Count == 0); }
public void Initialize(ConfigurationStorageAttribute config) { _config = config as CloudConfigurationStorageAttribute; if (_config == null) { return; } _connector = new S3Connector(_config.Location); }
public FileStreamResult downloadFile(string keyName) { GetObjectResponse result = S3Connector.ReadObjectData(WebUtility.UrlEncode(keyName)); return(new FileStreamResult(result.ResponseStream, new MediaTypeHeaderValue("text/plain")) { FileDownloadName = keyName, }); }
public async Task GetSingleFileTest() { var s3Connector = new S3Connector(region, bucketName); ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(from : new DateTime(2020, 05, 13)); // there is just one file from this date in S3 Assert.True(jobPostsKeys.Count == 1); }
public async Task GetAllFilesTest() { var s3Connector = new S3Connector(region, bucketName); ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(); // there are 2 folders with 1 file each Assert.True(jobPostsKeys.Count == 2); }
public static async Task Main() { var s3Connector = new S3Connector(RegionEndpoint.EUWest1, "jobposts-scraped"); IEnumerable <S3Object> jobPostsKeys = await s3Connector.GetFiles(to : new DateTime(2020, 05, 08)); IEnumerable <JobPost> jobPostsObjs = await s3Connector.GetFileContents(jobPostsKeys); JobPostRepo.Add(jobPostsObjs); Console.WriteLine("Starting the RUN()"); new Program().Run(); }
public static async Task Main(string[] args) { const string bucketName = "jobposts-scraped"; var s3Connector = new S3Connector(RegionEndpoint.EUWest1, bucketName); ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(from : DateTime.Now); ICollection <JobPost> jobPostsObjs = await s3Connector.GetFileContents(jobPostsKeys); JobPostRepo.Add(jobPostsObjs); Console.WriteLine("Starting the RUN()"); if (args.Length > 0 && !string.IsNullOrEmpty(args[0]) && !string.IsNullOrEmpty(args[1])) { await new Program().RunAsync(args[0], Convert.ToInt32(args[1])); } else { await new Program().RunAsync(); } }
public IActionResult UploadDocuments(IFormFile file) { Stream fileBuffer = file.OpenReadStream(); var db = new ClassroomContext(); Document doc = new Document(); doc.KeyName = WebUtility.UrlEncode(file.FileName); doc.FileName = file.FileName; if (file.ContentType.Contains("image")) { Stream fileBuffer2 = new MemoryStream(); fileBuffer.CopyTo(fileBuffer2); fileBuffer.Seek(0, SeekOrigin.Begin); IReadOnlyList <EntityAnnotation> resultText = GoogleMLConnector.ReadImageText(fileBuffer); fileBuffer = fileBuffer2; fileBuffer.Seek(0, SeekOrigin.Begin); IReadOnlyList <EntityAnnotation> resultLabels = GoogleMLConnector.LabelImage(fileBuffer); string lang = ""; string content = ""; foreach (var thing in resultText) { if (thing.Locale != null) { lang = thing.Locale; } content += thing.Description; } doc.Content = content; content = ""; foreach (var thing in resultLabels) { content += thing.Description + ", "; } doc.GoogleTags = content; fileBuffer = fileBuffer2; } else if (textValues.Contains <string>(file.ContentType)) { StreamReader reader = new StreamReader(fileBuffer); doc.Content = reader.ReadToEnd(); } else if (file.ContentType.ToLower().Contains("pdf")) { string cont = ""; PdfFixedDocument pdfdoc = new PdfFixedDocument(fileBuffer); List <PdfVisualImageCollection> img = new List <PdfVisualImageCollection>(); foreach (PdfPage page in pdfdoc.Pages) { PdfContentExtractor ce = new PdfContentExtractor(page); cont += ce.ExtractText(); //img.Add(ce.ExtractImages(true)); } doc.Content = cont; } else { StreamReader reader = new StreamReader(fileBuffer); doc.Content = reader.ReadToEnd(); } var uploadResult = S3Connector.UploadObject(file.FileName, fileBuffer, WebUtility.UrlEncode(file.FileName)); if (db.Documents.Where(x => x.KeyName == doc.KeyName).Count() == 0) { db.Documents.Add(doc); db.SaveChanges(); ElasticManager em = new ElasticManager(); em.addDocument(doc); } return(Content(doc.Id.ToString())); }
public ValuesController(S3Connector s3connector, DynamoDBConnector dynamoDBConnector, RekognitionConnector rekognitionConnector) { _s3Connector = s3connector; _dynamoDBConnector = dynamoDBConnector; _rekognitionConnector = rekognitionConnector; }
public void S3ConnectorCtorTest() { var s3Connector = new S3Connector(region, bucketName); Assert.False(s3Connector is null); }