public async Task GetFileContentsTest()
{
var s3Connector = new S3Connector(region, bucketName);
ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles();
ICollection <JobPost> jobPostsObjs = await s3Connector.GetFileContents(jobPostsKeys);
Assert.True(jobPostsObjs.Count == 2);
var javaJobPost = new JobPost("Senior Java Developer", "Start as soon as possible");
javaJobPost.Keywords.Add("java");
javaJobPost.Keywords.Add("maven");
javaJobPost.Keywords.Add("junit");
var cSharpJobPost = new JobPost("Senior Software Developer", "Contact us as soon as possible");
cSharpJobPost.Keywords.Add("c#");
cSharpJobPost.Keywords.Add("dotnet");
cSharpJobPost.Keywords.Add("full stack");
Assert.Equal(javaJobPost, jobPostsObjs.First(jobPost => jobPost.Header == javaJobPost.Header));
Assert.Equal(cSharpJobPost, jobPostsObjs.First(jobPost => jobPost.Header == cSharpJobPost.Header));
}
public async Task GetNoFilesTest()
{
var s3Connector = new S3Connector(region, bucketName);
ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(to : new DateTime(2020, 05, 08));
Assert.True(jobPostsKeys.Count == 0);
}
public void Initialize(ConfigurationStorageAttribute config)
{
_config = config as CloudConfigurationStorageAttribute;
if (_config == null)
{
return;
}
_connector = new S3Connector(_config.Location);
}
public FileStreamResult downloadFile(string keyName)
{
GetObjectResponse result = S3Connector.ReadObjectData(WebUtility.UrlEncode(keyName));
return(new FileStreamResult(result.ResponseStream, new MediaTypeHeaderValue("text/plain"))
{
FileDownloadName = keyName,
});
}
public async Task GetSingleFileTest()
{
var s3Connector = new S3Connector(region, bucketName);
ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(from : new DateTime(2020, 05, 13));
// there is just one file from this date in S3
Assert.True(jobPostsKeys.Count == 1);
}
public async Task GetAllFilesTest()
{
var s3Connector = new S3Connector(region, bucketName);
ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles();
// there are 2 folders with 1 file each
Assert.True(jobPostsKeys.Count == 2);
}
public static async Task Main()
{
var s3Connector = new S3Connector(RegionEndpoint.EUWest1, "jobposts-scraped");
IEnumerable <S3Object> jobPostsKeys = await s3Connector.GetFiles(to : new DateTime(2020, 05, 08));
IEnumerable <JobPost> jobPostsObjs = await s3Connector.GetFileContents(jobPostsKeys);
JobPostRepo.Add(jobPostsObjs);
Console.WriteLine("Starting the RUN()");
new Program().Run();
}
public static async Task Main(string[] args)
{
const string bucketName = "jobposts-scraped";
var s3Connector = new S3Connector(RegionEndpoint.EUWest1, bucketName);
ICollection <S3Object> jobPostsKeys = await s3Connector.GetFiles(from : DateTime.Now);
ICollection <JobPost> jobPostsObjs = await s3Connector.GetFileContents(jobPostsKeys);
JobPostRepo.Add(jobPostsObjs);
Console.WriteLine("Starting the RUN()");
if (args.Length > 0 && !string.IsNullOrEmpty(args[0]) && !string.IsNullOrEmpty(args[1]))
{
await new Program().RunAsync(args[0], Convert.ToInt32(args[1]));
}
else
{
await new Program().RunAsync();
}
}
public IActionResult UploadDocuments(IFormFile file)
{
Stream fileBuffer = file.OpenReadStream();
var db = new ClassroomContext();
Document doc = new Document();
doc.KeyName = WebUtility.UrlEncode(file.FileName);
doc.FileName = file.FileName;
if (file.ContentType.Contains("image"))
{
Stream fileBuffer2 = new MemoryStream();
fileBuffer.CopyTo(fileBuffer2);
fileBuffer.Seek(0, SeekOrigin.Begin);
IReadOnlyList <EntityAnnotation> resultText = GoogleMLConnector.ReadImageText(fileBuffer);
fileBuffer = fileBuffer2;
fileBuffer.Seek(0, SeekOrigin.Begin);
IReadOnlyList <EntityAnnotation> resultLabels = GoogleMLConnector.LabelImage(fileBuffer);
string lang = "";
string content = "";
foreach (var thing in resultText)
{
if (thing.Locale != null)
{
lang = thing.Locale;
}
content += thing.Description;
}
doc.Content = content;
content = "";
foreach (var thing in resultLabels)
{
content += thing.Description + ", ";
}
doc.GoogleTags = content;
fileBuffer = fileBuffer2;
}
else if (textValues.Contains <string>(file.ContentType))
{
StreamReader reader = new StreamReader(fileBuffer);
doc.Content = reader.ReadToEnd();
}
else if (file.ContentType.ToLower().Contains("pdf"))
{
string cont = "";
PdfFixedDocument pdfdoc = new PdfFixedDocument(fileBuffer);
List <PdfVisualImageCollection> img = new List <PdfVisualImageCollection>();
foreach (PdfPage page in pdfdoc.Pages)
{
PdfContentExtractor ce = new PdfContentExtractor(page);
cont += ce.ExtractText();
//img.Add(ce.ExtractImages(true));
}
doc.Content = cont;
}
else
{
StreamReader reader = new StreamReader(fileBuffer);
doc.Content = reader.ReadToEnd();
}
var uploadResult = S3Connector.UploadObject(file.FileName, fileBuffer, WebUtility.UrlEncode(file.FileName));
if (db.Documents.Where(x => x.KeyName == doc.KeyName).Count() == 0)
{
db.Documents.Add(doc);
db.SaveChanges();
ElasticManager em = new ElasticManager();
em.addDocument(doc);
}
return(Content(doc.Id.ToString()));
}
public ValuesController(S3Connector s3connector, DynamoDBConnector dynamoDBConnector, RekognitionConnector rekognitionConnector)
{
_s3Connector = s3connector;
_dynamoDBConnector = dynamoDBConnector;
_rekognitionConnector = rekognitionConnector;
}
public void S3ConnectorCtorTest()
{
var s3Connector = new S3Connector(region, bucketName);
Assert.False(s3Connector is null);
}