public async Task <DatasetModel> CreateOrUpdate(DatasetModel model)
{
ActivityDataset dataSet = null;
if (!model.Id.HasValue || model.Id == 0)
{
dataSet = _mapper.Map <ActivityDataset>(model);
_dbContext.Set <ActivityDataset>().Add(dataSet);
}
else
{
dataSet = await _dbContext.Set <ActivityDataset>().GetByIdAsync(model.Id.Value) ?? throw new AppException($"Dataset with Id: {model.Id} doesn't exist");
_mapper.Map(model, dataSet);
}
var newActivityLogs = _mapper.Map <List <ActivityLog> >(model.ActivityLogs);
newActivityLogs.ForEach(al => al.Dataset = dataSet);
_dbContext.Set <ActivityLog>().AddRange(newActivityLogs);
model.DeletedIds?.Select(id => new ActivityLog {
Id = id
})
.ToList().ForEach(e => _dbContext.Entry(e).State = EntityState.Deleted);
await _dbContext.SaveChangesAsync();
return(await _dbContext.Set <ActivityDataset>()
.Include(ad => ad.ActivityLogs)
.ProjectTo <DatasetModel>(_mapper.ConfigurationProvider)
.FirstOrDefaultAsync(ad => ad.Id == dataSet.Id));
}
public List <DatasetModel> ReadDataset()
{
string BaseDirectory = HttpContext.Current.Server.MapPath("/Dataset/" + ConfigurationManager.AppSettings["DatasetName"]);
string xmlDosyasi = BaseDirectory;
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.Load(xmlDosyasi);
List <DatasetModel> lst_data = new List <DatasetModel>();
foreach (XmlNode City in xmlDocument.DocumentElement.ChildNodes)
{
foreach (XmlNode district in City.ChildNodes)
{
foreach (XmlNode zipcode in district.ChildNodes)
{
DatasetModel datasetModel = new DatasetModel();
datasetModel.CityCode = City.Attributes["code"].Value;
datasetModel.CityName = City.Attributes["name"].Value;
datasetModel.DistrictName = district.Attributes["name"].Value;
datasetModel.ZipCode = zipcode.Attributes["code"].Value;
lst_data.Add(datasetModel);
}
}
}
return(lst_data);
}
private DatasetModel CreateDatasetModel(List <TimeDTO> times, List <DateTime> dates)
{
#region Datacheck
if (!times.Any())
{
throw new InvalidDataException("No data for series specified!");
}
if (times.Any(t => t.FK_D != times[0].FK_D))
{
throw new InvalidDataException("Multiple disciplines specified!");
}
#endregion
var serieColor = GenerateRandomColor();
var dataSet = new DatasetModel
{
Label = times.First().Person.DisplayName,
FillColor = serieColor,
StrokeColor = serieColor,
PointColor = serieColor,
PointStrokeColor = serieColor,
PointHighlightFill = serieColor,
PointHighlightStroke = "rgba(151,187,205,1)",
Data = new List <float>()
};
foreach (var time in times)
{
dataSet.Data.Add((float)time.Seconds);
}
return(dataSet);
}
public async Task DataSetCreationAndMetricsCalculation_Success()
{
var datasetsController = _serviceProvider.GetRequiredService <DatasetsController>();
var dataSetModel = new DatasetModel
{
Description = "test",
Name = "test",
ActivityLogs = Enumerable.Range(1, 10).Select(i => new ActivityLogModel
{
UserId = i,
RegistrationDate = new DateTime(2021, 1, i),
LastActivityDate = new DateTime(2021, 2, i)
}).ToList()
};
var creationResp = await datasetsController.CreateOrUpdate(dataSetModel);
var creationRes = creationResp.ExtractResponse();
var calcResp = await datasetsController.Calculate(creationRes.Data.Id.Value);
var calcRes = calcResp.ExtractResponse();
calcRes.Success.Should().BeTrue();
calcRes.Data.LifetimeChart.Should().BeEquivalentTo(new List <object> {
new { LifetimeInDays = 31, NumberOfUsers = 10 }
});
calcRes.Data.RollingRetention.Should().Be(100);
}
public async Task <IActionResult> Post([FromForm] DatasetModel model)
{
var data = await model.File.ReadAsListAsync();
await _datasetRepository.AddDatasetAsync(model.Name, data);
return(Ok());
}
public ActionResult SearchData(DatasetModel data)
{
if (data.keyword != null)
{
string logFilePath = Path.Combine(Server.MapPath(folderPath), logFileName);
var dataFilename = "";
// find the latest data from logFile
using (var sreader = new StreamReader(logFilePath))
{
dataFilename = sreader.ReadLine();
sreader.Close();
}
string savePath = Path.Combine(Server.MapPath(folderPath), dataFilename);
var dataset = new List <DataModel>();
// read and compare the data with the keyword
using (var sreader = new StreamReader(savePath))
{
while (!sreader.EndOfStream)
{
string[] rows = sreader.ReadLine().Split(',');
if (FindMatch(rows[1].ToString(), data.keyword))
{
// if match, store to a new list
dataset.Add(new DataModel
{
stringId = Guid.Parse(rows[0].ToString()),
stringContent = rows[1].ToString()
});
}
}
}
// extract stringId and add match count
var matchedStringId = (from d in dataset
select new DataModel
{
stringId = d.stringId,
matchCount = 1
}).ToList();
// update the match count and return totalCount
var newLogData = UpdateMatchCount(matchedStringId);
// assign the new matchCount to the current list
foreach (var item in dataset)
{
item.matchCount = (from a in newLogData
where a.stringId == item.stringId
select a.matchCount).First();
}
data.dataset = dataset;
}
return(View(data));
}
// GET: api/data/5
/// <summary>
/// In addition to the id, it is possible to have projection and selection criteria passed to the action via query string parameters
/// </summary>
/// <param name="id">Dataset Id</param>
/// <returns></returns>
/// <remarks> The action accepts the following additional parameters via the query string
/// 1: projection: is a comman separated list of ids that determines which variables of the dataset version tuples should take part in the result set
/// 2: selection: is a logical expression that filters the tuples of the chosen dataset. The expression should have been written against the variables of the dataset only.
/// logical operators, nesting, precedence, and SOME functions should be supported.
/// /api/data/6?header=TimeUTC,D8CO1_1&filter=TimeUTC<5706000
/// </remarks>
public HttpResponseMessage Get(int id)
{
string projection = this.Request.GetQueryNameValuePairs().FirstOrDefault(p => "header".Equals(p.Key, StringComparison.InvariantCultureIgnoreCase)).Value;
string selection = this.Request.GetQueryNameValuePairs().FirstOrDefault(p => "filter".Equals(p.Key, StringComparison.InvariantCultureIgnoreCase)).Value;
DatasetManager datasetManager = new DatasetManager();
try
{
OutputDataManager ioOutputDataManager = new OutputDataManager();
DatasetVersion version = datasetManager.GetDatasetLatestVersion(id);
string title = xmlDatasetHelper.GetInformationFromVersion(version.Id, NameAttributeValues.title);
// check the data sturcture type ...
if (version.Dataset.DataStructure.Self is StructuredDataStructure)
{
// apply selection and projection
//var tuples = dm.GetDatasetVersionEffectiveTuples(version);
DataTable dt = OutputDataManager.ConvertPrimaryDataToDatatable(datasetManager, version, title, true);
if (!string.IsNullOrEmpty(selection))
{
dt = OutputDataManager.SelectionOnDataTable(dt, selection);
}
if (!string.IsNullOrEmpty(projection))
{
// make the header names upper case to make them case insensitive
dt = OutputDataManager.ProjectionOnDataTable(dt, projection.ToUpper().Split(','));
}
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse();
response.Content = new ObjectContent(typeof(DatasetModel), model, new DatasetModelCsvFormatter(model.DataTable.TableName));
//set headers on the "response"
return(response);
//return model;
}
else
{
return(Request.CreateResponse());
}
}
finally
{
datasetManager.Dispose();
}
}
public async Task Post()
{
var controller = new DatasetController(datasetRepository);
var datasetModel = new DatasetModel()
{
Name = "dataset",
File = GetMockFile(),
};
var result = await controller.Post(datasetModel);
var okResult = Assert.IsType <OkResult>(result);
Assert.NotNull(okResult);
Assert.Equal(200, okResult.StatusCode);
}
public ActionResult SearchData()
{
var data = new DatasetModel();
return(View(data));
}
public SymptomsController()
{
dModel = new DatasetModel();
}
private HttpResponseMessage getData(long id, int variableId, string token)
{
DatasetManager datasetManager = new DatasetManager();
UserManager userManager = new UserManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
EntityManager entityManager = new EntityManager();
DataStructureManager dataStructureManager = null;
bool isPublic = false;
try
{
// if a dataset is public, then the api should also return data if there is no token for a user
#region is public
dataStructureManager = new DataStructureManager();
long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id;
entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1;
isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id);
#endregion is public
if (!isPublic && String.IsNullOrEmpty(token))
{
var request = Request.CreateResponse();
request.Content = new StringContent("Bearer token not exist.");
return(request);
}
User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault();
if (isPublic || user != null)
{
if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read))
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
OutputDataManager ioOutputDataManager = new OutputDataManager();
Dataset dataset = datasetManager.GetDataset(id);
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
object stats = new object();
DataTable dt = new DataTable("Varibales");
List <ApiDataStatisticModel> dataStatisticModels = new List <ApiDataStatisticModel>();
if (variableId == -1)
{
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id);
List <string> varIds = new List <string>();
foreach (Variable vs in structuredDataStructure.Variables)
{
varIds.Add("var" + vs.Id);
}
dt = GetDuplicates(id, varIds);
}
else
{
}
//dt.Strip();
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse(HttpStatusCode.OK);
string resp = JsonConvert.SerializeObject(model);
response.Content = new StringContent(resp, System.Text.Encoding.UTF8, "application/json");
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return(response);
}
else
{
return(Request.CreateResponse());
}
}
else // has rights?
{
var request = Request.CreateResponse();
request.Content = new StringContent("User has no read right.");
return(request);
}
}
else
{
var request = Request.CreateResponse();
request.Content = new StringContent("User is not available.");
return(request);
}
}
catch (Exception e)
{
throw e;
}
finally
{
datasetManager.Dispose();
userManager.Dispose();
entityPermissionManager.Dispose();
entityManager.Dispose();
dataStructureManager.Dispose();
}
}
private HttpResponseMessage getData(long id, int variableId, string token)
{
DatasetManager datasetManager = new DatasetManager();
UserManager userManager = new UserManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
EntityManager entityManager = new EntityManager();
DataStructureManager dataStructureManager = null;
bool isPublic = false;
try
{
// if a dataset is public, then the api should also return data if there is no token for a user
#region is public
dataStructureManager = new DataStructureManager();
long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id;
entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1;
isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id);
#endregion is public
if (!isPublic && String.IsNullOrEmpty(token))
{
var request = Request.CreateResponse();
request.Content = new StringContent("Bearer token not exist.");
return(request);
}
User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault();
if (isPublic || user != null)
{
if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read))
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
OutputDataManager ioOutputDataManager = new OutputDataManager();
Dataset dataset = datasetManager.GetDataset(id);
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
object stats = new object();
DataTable dt = new DataTable("Varibales");
DataTable dtMissingValues = new DataTable("MissingValues");
dtMissingValues.Columns.Add("placeholder", typeof(String));
dtMissingValues.Columns.Add("displayName", typeof(String));
List <ApiDataStatisticModel> dataStatisticModels = new List <ApiDataStatisticModel>();
StructuredDataStructure structuredDataStructure = dataStructureManager.StructuredDataStructureRepo.Get(datasetVersion.Dataset.DataStructure.Id);
if (variableId == -1)
{
foreach (Variable vs in structuredDataStructure.Variables)
{
ApiDataStatisticModel dataStatisticModel = new ApiDataStatisticModel();
dt = GetUniqueValues(id, vs.Id);
dataStatisticModel.VariableId = vs.Id;
dataStatisticModel.uniqueValues = dt;
dataStatisticModel.minLength = dt.Compute("Min(length)", string.Empty).ToString();
dataStatisticModel.maxLength = dt.Compute("Max(length)", string.Empty).ToString();
dataStatisticModel.count = dt.Compute("Sum(count)", string.Empty).ToString();
dtMissingValues.Clear();
foreach (var missingValue in vs.MissingValues)
{
DataRow workRow = dtMissingValues.NewRow();
workRow["placeholder"] = missingValue.Placeholder;
workRow["displayName"] = missingValue.DisplayName;
dtMissingValues.Rows.Add(workRow);
}
dataStatisticModel.min = GetMin(dtMissingValues, dt);
dataStatisticModel.max = GetMax(dtMissingValues, dt);
dataStatisticModel.missingValues = dtMissingValues;
dataStatisticModels.Add(dataStatisticModel);
}
}
else
{
Variable variable = new Variable();
foreach (Variable vs in structuredDataStructure.Variables)
{
if (vs.Id == variableId)
{
variable = vs;
}
}
ApiDataStatisticModel dataStatisticModel = new ApiDataStatisticModel();
dt = GetUniqueValues(id, variableId);
dataStatisticModel.VariableId = variableId;
dataStatisticModel.uniqueValues = dt;
dataStatisticModel.minLength = dt.Compute("Min(length)", string.Empty).ToString();
dataStatisticModel.maxLength = dt.Compute("Max(length)", string.Empty).ToString();
dataStatisticModel.count = dt.Compute("Sum(count)", string.Empty).ToString();
foreach (var missingValue in variable.MissingValues)
{
DataRow workRow = dtMissingValues.NewRow();
workRow["placeholder"] = missingValue.Placeholder;
workRow["displayName"] = missingValue.DisplayName;
dtMissingValues.Rows.Add(workRow);
}
dataStatisticModel.min = GetMin(dtMissingValues, dt);
dataStatisticModel.max = GetMax(dtMissingValues, dt);
dataStatisticModel.missingValues = dtMissingValues;
dataStatisticModels.Add(dataStatisticModel);
}
dt.Strip();
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse(HttpStatusCode.OK);
string resp = JsonConvert.SerializeObject(dataStatisticModels);
response.Content = new StringContent(resp, System.Text.Encoding.UTF8, "application/json");
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return(response);
}
else
{
return(Request.CreateResponse());
}
}
else // has rights?
{
var request = Request.CreateResponse();
request.Content = new StringContent("User has no read right.");
return(request);
}
}
else
{
var request = Request.CreateResponse();
request.Content = new StringContent("User is not available.");
return(request);
}
}
catch (Exception e)
{
throw e;
}
finally
{
datasetManager.Dispose();
userManager.Dispose();
entityPermissionManager.Dispose();
entityManager.Dispose();
dataStructureManager.Dispose();
}
}
public async Task <ActionResult <ResponseVm <DatasetModel> > > CreateOrUpdate(DatasetModel model)
{
return(ApiResponse(await _datasetService.CreateOrUpdate(model)));
}
private HttpResponseMessage getData(long id, int version, string token, string projection = null, string selection = null)
{
DatasetManager datasetManager = new DatasetManager();
UserManager userManager = new UserManager();
EntityPermissionManager entityPermissionManager = new EntityPermissionManager();
EntityManager entityManager = new EntityManager();
bool isPublic = false;
try
{
// if a dataset is public, then the api should also return data if there is no token for a user
#region is public
long?entityTypeId = entityManager.FindByName(typeof(Dataset).Name)?.Id;
entityTypeId = entityTypeId.HasValue ? entityTypeId.Value : -1;
isPublic = entityPermissionManager.Exists(null, entityTypeId.Value, id);
#endregion is public
if (!isPublic && String.IsNullOrEmpty(token))
{
var request = Request.CreateResponse();
request.Content = new StringContent("Bearer token not exist.");
return(request);
}
User user = userManager.Users.Where(u => u.Token.Equals(token)).FirstOrDefault();
if (isPublic || user != null)
{
if (isPublic || entityPermissionManager.HasEffectiveRight(user.Name, typeof(Dataset), id, RightType.Read))
{
XmlDatasetHelper xmlDatasetHelper = new XmlDatasetHelper();
OutputDataManager ioOutputDataManager = new OutputDataManager();
Dataset dataset = datasetManager.GetDataset(id);
// If the requested version is -1 or the last version of the dataset, then the data will be loaded in a
// different way than when loading the data from an older version
bool isLatestVersion = false;
if (version == -1 || dataset.Versions.Count == version)
{
isLatestVersion = true;
}
if (isLatestVersion)
{
#region get data from the latest version of a dataset
DatasetVersion datasetVersion = datasetManager.GetDatasetLatestVersion(id);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
//FilterExpression filter = null;
//OrderByExpression orderBy = null;
//ProjectionExpression projectionExpression = GetProjectionExpression(projection);
// apply selection and projection
long count = datasetManager.RowCount(id);
DataTable dt = datasetManager.GetLatestDatasetVersionTuples(id, null, null, null, 0, (int)count);
dt.Strip();
if (!string.IsNullOrEmpty(selection))
{
dt = OutputDataManager.SelectionOnDataTable(dt, selection, true);
}
if (!string.IsNullOrEmpty(projection))
{
// make the header names upper case to make them case insensitive
dt = OutputDataManager.ProjectionOnDataTable(dt, projection.ToUpper().Split(','));
}
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse();
response.Content = new ObjectContent(typeof(DatasetModel), model, new DatasetModelCsvFormatter(model.DataTable.TableName));
response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/csv");
//set headers on the "response"
return(response);
#endregion get data from the latest version of a dataset
//return model;
}
else
{
return(Request.CreateResponse());
}
}
else
{
#region load data of a older version of a dataset
int index = version - 1;
if (version >= dataset.Versions.Count)
{
return(Request.CreateResponse(HttpStatusCode.PreconditionFailed, String.Format("This version ({0}) is not available for the dataset", version)));
}
DatasetVersion datasetVersion = dataset.Versions.OrderBy(d => d.Timestamp).ElementAt(version - 1);
string title = datasetVersion.Title;
// check the data sturcture type ...
if (datasetVersion.Dataset.DataStructure.Self is StructuredDataStructure)
{
//FilterExpression filter = null;
//OrderByExpression orderBy = null;
// apply selection and projection
int count = datasetManager.GetDatasetVersionEffectiveTuples(datasetVersion).Count;
DataTable dt = datasetManager.GetDatasetVersionTuples(datasetVersion.Id, 0, count);
dt.Strip();
if (!string.IsNullOrEmpty(selection))
{
dt = OutputDataManager.SelectionOnDataTable(dt, selection);
}
if (!string.IsNullOrEmpty(projection))
{
// make the header names upper case to make them case insensitive
dt = OutputDataManager.ProjectionOnDataTable(dt, projection.ToUpper().Split(','));
}
dt.TableName = id + "_data";
DatasetModel model = new DatasetModel();
model.DataTable = dt;
var response = Request.CreateResponse();
response.Content = new ObjectContent(typeof(DatasetModel), model, new DatasetModelCsvFormatter(model.DataTable.TableName));
response.Content.Headers.ContentType = new MediaTypeHeaderValue("text/csv");
//set headers on the "response"
return(response);
}
else // return files of the unstructure dataset
{
return(Request.CreateResponse());
}
#endregion load data of a older version of a dataset
}
}
else // has rights?
{
var request = Request.CreateResponse();
request.Content = new StringContent("User has no read right.");
return(request);
}
}
else
{
var request = Request.CreateResponse();
request.Content = new StringContent("User is not available.");
return(request);
}
}
catch (Exception e)
{
throw e;
}
finally
{
datasetManager.Dispose();
userManager.Dispose();
entityPermissionManager.Dispose();
entityManager.Dispose();
}
}
