public async Task <IActionResult> OnPostAsync()
{
if (!ModelState.IsValid)
{
return(Page());
}
var BlobName = Request.Form["blobName"];
var BlobKey = Request.Form["blobKey"];
var BlobContainer = Request.Form["blobContainer"];
var CosmosUrl = Request.Form["cosmosUrl"];
var CosmosKey = Request.Form["cosmosKey"];
var DatabaseName = Request.Form["OutputDB"];
var CollectionName = Request.Form["OutputColl"];
var Path = Request.Form["folder"];
BlobManager blobManager = new BlobManager(BlobName, BlobKey);
List <IListBlobItem> items = blobManager.ListAllBlobsAsync(BlobContainer, Path).Result;
_logger.LogInformation($"{items.Count} items found to import");
List <TraveltimeSegmentStatic> segments = new List <TraveltimeSegmentStatic>();
foreach (IListBlobItem item in items)
{
segments.AddRange(CSVReader.GetStaticsFromCSVString(blobManager.DownloadBlob(item.Container.Name, BlobManager.getBlobIdFromURI(item.Uri)).Result));
}
_logger.LogInformation($"{segments.Count} documents parsed");
CrudManager <TraveltimeSegmentStatic> crudManager = new CrudManager <TraveltimeSegmentStatic>
{
DatabaseUri = CosmosUrl,
DatabaseKey = CosmosKey,
DatabaseName = DatabaseName,
CollectionName = CollectionName
};
crudManager.Init();
foreach (TraveltimeSegmentStatic segment in segments)
{
await crudManager.Create(segment);
}
return(RedirectToPage("/BE-Mobile/BEM_import_static"));
}
public async Task Import(TaskModel model)
{
if (model.BlobInput == null)
{
Console.WriteLine("No Storage account connection info found, skipping task");
return;
}
Console.WriteLine($"Starting blob import task for {model.BlobInput.AccountName},{model.BlobInput.Container},{model.BlobInput.Key} path {model.BlobInput.Path}");
BlobManager manager = new BlobManager(model.BlobInput.AccountName, model.BlobInput.Key);
List <Task> runningUploadTasks = new List <Task>();
foreach (var path in model.BlobInput.Path)
{
Console.WriteLine($"Fetching al blobs at path {path}");
List <IListBlobItem> items = manager.ListAllBlobsAsync(model.BlobInput.Container, path).Result;
Console.WriteLine($"Found {items.Count} items at path {path}, parsing...");
List <TraveltimeSegment> segments = new List <TraveltimeSegment>();
foreach (IListBlobItem item in items)
{
segments.AddRange(CSVReader.GetSegmentsFromCSVString(manager.DownloadBlob(item.Container.Name, BlobManager.getBlobIdFromURI(item.Uri)).Result));
}
CrudManager <TraveltimeSegment> crudManager = new CrudManager <TraveltimeSegment>
{
DatabaseKey = model.Output.Key,
DatabaseUri = model.Output.URL,
DatabaseName = model.Output.Database,
CollectionName = model.Output.Collection,
};
crudManager.Init();
TotalToProcess += segments.Count;
runningUploadTasks.Add(UploadToCosmos(crudManager, segments));
}
await Task.WhenAll();
}
public void Execute(ILogger logger, IBlockingQueue <BeMobileTaskModel, BEMobileSegmentTaskModel> queue)
{
if (BlobName == null && BlobKey == null)
{
logger.LogWarning("Execute called upon empty task");
return;
}
logger.LogInformation("Starting BE-Mobile import task");
BlobManager manager = new BlobManager(BlobName, BlobKey);
List <IListBlobItem> items = manager.ListAllBlobsAsync(BlobContainer, $"BE-Mobile/{Month}/Reistijden/{Traject}/").Result;
CrudManager <TraveltimeSegment> crudManager = new CrudManager <TraveltimeSegment>
{
DatabaseKey = CosmosKey,
DatabaseUri = CosmosUrl,
DatabaseName = DatabaseName,
CollectionName = CollectionName,
};
crudManager.Init();
List <TraveltimeSegment> segments = new List <TraveltimeSegment>();
foreach (IListBlobItem item in items)
{
segments.AddRange(CSVReader.GetSegmentsFromCSVString(manager.DownloadBlob(item.Container.Name, BlobManager.getBlobIdFromURI(item.Uri)).Result));
}
foreach (TraveltimeSegment segment in segments)
{
queue.DocumentQueue.Add(new BEMobileSegmentTaskModel
{
CreationMethod = crudManager.Create,
SegmentToCreate = segment
});
}
queue.DocumentsFinished += 1;
}
