public IList <dynamic> LoadCSVFiles <T>(IList <string> filepaths, Job <T> jobObj, int blocksize = 0, int sleepSecs = 0, uint chunkMB = 0) where T : StreamFormat
{
if (blocksize == 0)
{
blocksize = filepaths.Count;
}
else if (blocksize < MINFILEBLOCK)
{
blocksize = MINFILEBLOCK;
}
if (sleepSecs < MINSLEEPSEC)
{
sleepSecs = MINSLEEPSEC;
}
List <dynamic> jobResponses = new List <dynamic>();
dynamic jobResponse = null;
var jobid = "";
for (int i = 0; i < filepaths.Count; i++)
{
if (i % blocksize == 0)
{
// Create job
try
{
jobResponse = _endpointUri
.AppendPathSegments("accounts", _account, "jobs")
.WithOAuthBearerToken(_token)
.PostJsonAsync(jobObj)
.ReceiveJson().Result;
// var jsonResponse = JsonConvert.SerializeObject(jobResponse);
jobid = jobResponse.id;
var clone = Clone(jobResponse);
// Callback
_responseCallback?.Invoke(clone);
// Add to responses
jobResponses.Add(clone);
}
catch (Exception e)
{
// TODO logging
// Unwrappingnes
if (e is System.AggregateException)
{
foreach (var ie in ((System.AggregateException)e).InnerExceptions)
{
if (ie is FlurlHttpException)
{
string msg = ((FlurlHttpException)ie).GetResponseStringAsync().Result;
ie.HelpLink = msg;
}
}
}
e.HelpLink = $"failure to create job starting with file {filepaths[i]} with blocksize {blocksize}";
var excResponse = CreateExceptionResponse(e, "creating job");
// Call callback
_responseCallback?.Invoke(excResponse);
jobResponses.Add(excResponse);;
break;
}
}
// Send file
var start = DateTime.Now;
try
{
var info = new FileInfo(filepaths[i]);
var size = info.Length;
Console.WriteLine($"Sending file {filepaths[i]} of {info.Length} bytes" + (chunkMB > 0?$", in chunks of {chunkMB} MB":""));
var fileResponses = new List <dynamic>();
// Loop sending chunks
// Send data
//curl -X POST --header 'Content-type: text/csv' --header 'Authorization: Bearer [token]' --data-binary "@/Users/User1/Desktop/HumanActivity/source1.csv" 'http://<serveraddress>:30063/api/1.1/accounts/1549746082718454/datastreams/1554938538981549/ingestdata/1554942688319300/inputs
using (var reader = File.OpenText(filepaths[i]))
{
// If no chunking send entire content at once
if (chunkMB <= 0)
{
var fileContent = new StringContent(File.ReadAllText(filepaths[i]));
var fileResponse = new Url(_baseUri + jobResponse.links[0].url)
.WithHeader("Content-type", "text/csv")
.WithOAuthBearerToken(_token)
.PostAsync(fileContent)
.ReceiveJson().Result;
fileResponses.Add(fileResponse);
// Call callback
_responseCallback?.Invoke(fileResponse);
}
else
{
Action <String> sendText = delegate(String text)
{
var chunkContent = new StringContent(text);
var chunkResponse = new Url(_baseUri + jobResponse.links[0].url)
.WithHeader("Content-type", "text/csv")
.WithOAuthBearerToken(_token)
.PostAsync(chunkContent)
.ReceiveJson().Result;
fileResponses.Add(chunkResponse);
// Call callback
_responseCallback?.Invoke(chunkResponse);
};
Chunker.ChunkTextFromText(Chunker.GetChunkSize(chunkMB), reader, sendText);
}
}
// TODO: Move to callbacks
dynamic stats = new ExpandoObject();
stats.jobid = jobid;
stats.file = filepaths[i];
stats.time = TimeSpan.FromTicks(DateTime.Now.Ticks - start.Ticks);
Console.WriteLine($"It took {stats.time} to send file {stats.file}");
// Add to responses
jobResponses.Add(stats);
jobResponses.AddRange(fileResponses);
}
catch (Exception e)
{
// TODO logging
// Unwrapping
if (e is System.AggregateException)
{
foreach (var ie in ((System.AggregateException)e).InnerExceptions)
{
if (ie is FlurlHttpException)
{
string msg = ((FlurlHttpException)ie).GetResponseStringAsync().Result;
ie.HelpLink = msg;
}
}
}
e.HelpLink = $"failure to send file {filepaths[i]} after {TimeSpan.FromTicks(DateTime.Now.Ticks-start.Ticks)}";
var excResponse = CreateExceptionResponse(e, "sending file");
// Call callback for exception thrown
_responseCallback?.Invoke(excResponse);
jobResponses.Add(excResponse);
}
if (i % blocksize == (blocksize - 1) || i == (filepaths.Count - 1))
{
try
{
// Complete files ingest job
var complJobObj = new Job <T>();
complJobObj.jobType = JOBTYPE.INGESTDATA;
complJobObj.status = JOBSTATUS.COMPLETED;
complJobObj.datastream = jobObj.datastream;
complJobObj.spec = jobObj.spec;
jobResponse = _endpointUri
.AppendPathSegments("accounts", _account, "jobs", (string)jobResponse.id)
.WithOAuthBearerToken(_token)
.PutJsonAsync(complJobObj)
.ReceiveJson().Result;
// Call callback
_responseCallback?.Invoke(jobResponse);
// Add to responses
jobResponses.Add(Clone(jobResponse));
}
catch (Exception e)
{
// TODO logging
// Unwrapping
if (e is System.AggregateException)
{
foreach (var ie in ((System.AggregateException)e).InnerExceptions)
{
if (ie is FlurlHttpException)
{
string msg = ((FlurlHttpException)ie).GetResponseStringAsync().Result;
ie.HelpLink = msg;
}
}
}
e.HelpLink = $"failure to close the job {(string)jobResponse.id} starting with file {filepaths[i - (blocksize - 1)]} with blocksize {blocksize}";
var excResponse = CreateExceptionResponse(e, "closing job");
// Call callback
_responseCallback?.Invoke(excResponse);
// Add to responses
jobResponses.Add(excResponse);
break;
}
if (sleepSecs > 0)
{
Thread.Sleep(sleepSecs * 1000);
}
}
}
return(jobResponses);
}
//curl -X POST --header 'Content-type: text/csv' --header 'Authorization: Bearer [token]' --data-binary "@/Users/User1/Desktop/HumanActivity/source1.csv" 'http://<serveraddress>:30063/api/1.1/accounts/1549746082718454/datastreams/1554938538981549/ingestdata/1554942688319300/inputs'
public IList <dynamic> LoadCSVFile <T>(string filepath, Job <T> jobObj, uint chunkMB = 0) where T : StreamFormat
{
// Create Job
//var jobInJson = JsonConvert.SerializeObject(jobObj);
var jobResponses = new List <dynamic>();
try
{
var jobResponse = _endpointUri
.AppendPathSegments("accounts", _account, "jobs")
.WithOAuthBearerToken(_token)
.PostJsonAsync(jobObj)
.ReceiveJson().Result;
jobResponses.Add(jobResponse);
// var jsonResponse = JsonConvert.SerializeObject(jobResponse);
// Loop sending chunks
// Send data
//curl -X POST --header 'Content-type: text/csv' --header 'Authorization: Bearer [token]' --data-binary "@/Users/User1/Desktop/HumanActivity/source1.csv" 'http://<serveraddress>:30063/api/1.1/accounts/1549746082718454/datastreams/1554938538981549/ingestdata/1554942688319300/inputs
using (var reader = File.OpenText(filepath))
{
// If no chunking send entire content at once
if (chunkMB <= 0)
{
var fileContent = new StringContent(File.ReadAllText(filepath));
var fileResponse = new Url(_baseUri + jobResponse.links[0].url)
.WithHeader("Content-type", "text/csv")
.WithOAuthBearerToken(_token)
.PostAsync(fileContent)
.ReceiveJson().Result;
jobResponses.Add(fileResponse);
}
else
{
Action <String> sendText = delegate(String text)
{
var chunkContent = new StringContent(text);
var chunkResponse = new Url(_baseUri + jobResponse.links[0].url)
.WithHeader("Content-type", "text/csv")
.WithOAuthBearerToken(_token)
.PostAsync(chunkContent)
.ReceiveJson().Result;
jobResponses.Add(chunkResponse);
};
Chunker.ChunkTextFromText(Chunker.GetChunkSize(chunkMB), reader, sendText);
}
}
// Complete file ingest job
var complJobObj = new Job <T>();
complJobObj.jobType = JOBTYPE.INGESTDATA;
complJobObj.status = JOBSTATUS.COMPLETED;
complJobObj.datastream = jobObj.datastream;
complJobObj.spec = jobObj.spec;
jobResponse = _endpointUri
.AppendPathSegments("accounts", _account, "jobs", (string)jobResponse.id)
.WithOAuthBearerToken(_token)
.PutJsonAsync(complJobObj)
.ReceiveJson().Result;
jobResponses.Add(jobResponse);
}
catch (Exception e)
{
// TODO logging
// Unwrapping
if (e is System.AggregateException)
{
foreach (var ie in ((System.AggregateException)e).InnerExceptions)
{
if (ie is FlurlHttpException)
{
string msg = ((FlurlHttpException)ie).GetResponseStringAsync().Result;
ie.HelpLink = msg;
}
}
}
throw (e);
}
return(jobResponses);
}
