public DataFile CreateFolder(int projectId, DirectoryDTO folder)
{
//if (Directory.Exists(newDir))
//return new DirectoryInfo(newDir);
//var di = Directory.CreateDirectory(newDir);
var project = _projectRepository.FindSingle(p => p.Id == projectId);
if (project == null || folder.Name == "" || folder.Name == null)
{
return(null);
}
var file = new DataFile();
file.FileName = folder.Name;
file.DateAdded = DateTime.Now.ToString("D");
file.IsDirectory = true;
file.ProjectId = project.Id;
if (folder.ParentFolderId != 0)
{
file.FolderId = folder.ParentFolderId;
}
_fileRepository.Insert(file);
return(_dataServiceUnit.Save().Equals("CREATED") ? file : null);
}
public DirectoryInfo AddDirectory(int projectId, string newDir)
{
if (Directory.Exists(newDir))
{
return(new DirectoryInfo(newDir));
}
var di = Directory.CreateDirectory(newDir);
var project = _projectRepository.FindSingle(p => p.Id == projectId);
if (project == null)
{
return(null);
}
var file = new DataFile();
file.FileName = di.Name;
//file.Path = di.FullName.Substring(di.FullName.IndexOf(projectId));
file.Path = di.Parent.FullName.Substring(di.Parent.FullName.IndexOf("P-" + projectId));//file.Path.Substring(0,file.Path.LastIndexOf("\\\"))
file.DateAdded = di.CreationTime.ToString("D");
file.IsDirectory = true;
file.ProjectId = project.Id;
_fileRepository.Insert(file);
return(_dataServiceUnit.Save().Equals("CREATED") ? di : null);
}
public Dataset addDataset(DatasetDTO datasetDTO)
{
//1. Fields for Dataset
var dataset = CreateDataset(datasetDTO);
_datasetRepository.Insert(dataset);
return(_dataServiceUnit.Save().Equals("CREATED") ? dataset : null);
}
public void DeleteProject(int projectId)
{
var projectToDelete = _projectRepository.Get(projectId);
_projectRepository.Remove(projectToDelete);
_sdtmRepository.DeleteMany(s => s.ProjectId == projectId);
uoW.Save();
}
public AssayDTO AddAssay(AssayDTO assayDto)
{
var assay = new Assay();
//var project = _projectRepository.FindSingle(d => d.Accession
// .Equals(assayDto.ProjectAcc));
assay.Name = assayDto.Name;
assay.ProjectId = assayDto.ProjectId;
assay.TechnologyPlatformId = assayDto.Platform;
assay.TechnologyTypeId = assayDto.Technology;
//assay.DesignType = getCVterm(assayDto.AssayDesignType);
assay.MeasurementTypeId = assayDto.Type;
if (assayDto.SamplesDataset != null)
{
assayDto.SamplesDataset.ProjectId = assayDto.ProjectId;
}
if (assayDto.FeaturesDataset != null)
{
assayDto.FeaturesDataset.ProjectId = assayDto.ProjectId;
}
if (assayDto.ObservationsDataset != null)
{
assayDto.ObservationsDataset.ProjectId = assayDto.ProjectId;
}
var BSdataset = _datasetService.CreateDataset(assayDto.SamplesDataset);
if (BSdataset != null)
{
assay.Datasets.Add(BSdataset);
}
var FEdataset = _datasetService.CreateDataset(assayDto.FeaturesDataset);
if (FEdataset != null)
{
assay.Datasets.Add(FEdataset);
}
var OBdataset = _datasetService.CreateDataset(assayDto.ObservationsDataset);
if (OBdataset != null)
{
assay.Datasets.Add(OBdataset);
}
assay = _assayRepository.Insert(assay);
if (_dataContext.Save().Equals("CREATED"))
{
assayDto.Id = assay.Id;
return(assayDto);
}
return(null);
}
//public void addDatasetVariables(List<VariableDefinition> variableDefinitions)
//{
// for (int i = 0; i < variableDefinitions.Count; i++)
// {
// _variableDefinitionRepository.Insert(variableDefinitions[i]);
// }
// _studyServiceUnit.Save();
//}
public string Updatestudy(StudyDTO studyDto, int studyId)
{
var studyToUpdate = _studyRepository.Get(studyId);
studyToUpdate.Name = studyDto.Name;
studyToUpdate.Description = studyDto.Title;
_studyRepository.Update(studyToUpdate);
return(_studyServiceUnit.Save());
}
public void LoadDictionaries()
{
var reader = File.OpenText(Path.Combine(_JSONDir, _termsJSONfilename));
List <Dictionary> dicts = JsonConvert.DeserializeObject <List <Dictionary> >(reader.ReadToEnd());
_dictRepository.InsertMany(dicts);
_db.Save();
reader.Dispose();
}
public ActivityDTO AddActivity(ActivityDTO activityDTO)
{
//var project = _projectRepository.FindSingle(d => d.Accession
// .Equals(activityDTO.ProjectAcc));
var activity = new Activity {
Name = activityDTO.Name, ProjectId = activityDTO.ProjectId
};
foreach (var datasetDto in activityDTO.datasets)
{
datasetDto.ProjectId = activityDTO.ProjectId;
var dataset = _datasetService.CreateDataset(datasetDto);
activity.Datasets.Add(dataset);
}
activity = _activityRepository.Insert(activity);
if (_activityServiceUnit.Save().Equals("CREATED"))
{
activityDTO.Id = activity.Id;
return(activityDTO);
}
return(null);
}
public UserDatasetDTO AddUserDataset(UserDatasetDTO dto, string userId)
{
var NewDS = ReadDTO(dto);
NewDS.OwnerId = userId;
//var project = _projectRepository.FindSingle(p => p.Accession == dto.ProjectAcc);
NewDS.ProjectId = dto.ProjectId;
NewDS.Id = Guid.NewGuid();
var addedUserDataset = _userDatasetRepository.Insert(NewDS);
if (!_unitOfWork.Save().Equals("CREATED"))
{
return(null);
}
dto.Id = addedUserDataset.Id.ToString();
return(dto);
}
internal bool LoadHDdata(int datasetId, int fileId, DataTable dataTable)
{
var dataset = _datasetRepository.FindSingle(d => d.Id == datasetId);
var dataFile = _dataFileRepository.Get(fileId);
//if (!dataFile.State.ToLower().Equals("new"))
//{
// /**
// * Replacing previously loaded file
// * Remove file from collection before reloading it
// */
// _observationRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId);
// Debug.WriteLine("RECORD(s) SUCCESSFULLY DELETED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId);
//}
var projectId = dataFile.ProjectId;
var observations = new List <Observation>();
var totalLoaded = 0.0;
var prop = new PropertyDescriptor()
{
Name = "CONC",
ProjectId = projectId,
Description = "Concentration",
ObsClass = "ASSAYMEASURES",
Id = 99
};
try
{
foreach (DataRow row in dataTable.Rows)
{
var obs = new Observation();
obs.FeatureName = row["FEAT"]?.ToString();
// obs.SubjectOfObservationId = row["SPECIMEN"]?.ToString();
obs.SubjectOfObservationName = row["SPECIMEN"]?.ToString();
double val;
if (double.TryParse(row["OBSVAL"]?.ToString(), out val))
{
obs.ObservedValue = new NumericalValue()
{
Property = prop,
Unit = row["OBSVALU"]?.ToString(),
Value = val
}
}
;
else
{
obs.ObservedValue = new MissingValue()
{
Property = prop
};
}
obs.Id = Guid.NewGuid();
obs.DatasetId = datasetId;
obs.ActivityId = dataset.ActivityId;
obs.DatafileId = fileId;
obs.ProjectId = projectId;
observations.Add(obs);
if (observations.Count % 500 == 0)
{
_observationRepository.InsertMany(observations);
totalLoaded += observations.Count;
dataFile.State = Math.Round(totalLoaded / dataTable.Rows.Count * 100).ToString("##");
_dataFileRepository.Update(dataFile);
_dataContext.Save();
observations.Clear();
}
}
_observationRepository.InsertMany(observations);
Debug.WriteLine(dataTable.Rows.Count + " RECORD(s) SUCCESSFULLY ADDED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId);
}
catch (Exception)
{
dataFile.State = "FAILED";
dataFile.IsLoadedToDB = false;
_dataFileRepository.Update(dataFile);
_dataContext.Save();
return(false);
}
dataFile.State = "LOADED";
dataFile.IsLoadedToDB = true;
_dataFileRepository.Update(dataFile);
_dataContext.Save();
return(true);
}
// dataset service
public bool LoadHDdDdata(int datasetId, int fileId /*, int referencFromHdId*/)
{
_observationRepository.DeleteMany(d => d.DatasetId == datasetId);
var dataset = GetActivityDataset(datasetId);
var dataFile = _dataFileRepository.Get(fileId);
var filePath = dataFile.Path + "\\" + dataFile.FileName;
var dataTable = _fileService.ReadOriginalFile(filePath);
var sampleList = _biosampleRepository.FindAll(s => s.DatasetId == datasetId).ToList();
var scos = sampleList.ToDictionary(co => co.BiosampleStudyId);
var obsReadyToInsert = new List <Observation>();
foreach (DataRow row in dataTable.Rows)
{
for (int index = 0; index < dataTable.Columns.Count; index++)
{
if (index == 0)
{
continue;
}
else
{
var column = dataTable.Columns[index];
var obs = new Observation();
// Fill an Observation
var value = new NumericalValue();
value.Value = double.Parse(row[column.ColumnName].ToString());
value.Property = new PropertyDescriptor();
{
value.Property.Description = dataset.Template.Description;
value.Property.ObsClass = dataset.Template.Class;
value.Property.Name = dataset.TemplateId;
}
// value.Unit = ??
obs.ObservedValue = value; // Observed Value which includes in it the VALUE and its PROPERTY
// ****** FOR NOW WE USE THE FOLLOWING TILL WE HAVE THE FEATURE referencFromHdId ******
obs.SubjectOfObservationName = column.ColumnName; // sample name (HERE IS THE FILE NAME)
//obs.SubjectOfObservationId = column.ColumnName; // sampleID (here is the file name)
// ******
// obs.SubjectOfObservationName = scos[column.ColumnName].BiosampleStudyId; // sample Name
// obs.SubjectOfObservationId = scos[column.ColumnName].Id.ToString(); // sample ID
// obs.StudyId = scos[column.ColumnName].StudyId; // study ID
obs.FeatureName = row[0].ToString(); // feature name
// obs.FeatureId =; Not implemented yet // feature ID
obs.DatafileId = dataFile.Id;
obs.DatasetId = datasetId;
obs.ActivityId = dataset.ActivityId;
obs.ProjectId = dataset.Activity.ProjectId;
obs.Id = Guid.NewGuid();
obsReadyToInsert.Add(obs);
// observationsTotal.Add(obs);
if (obsReadyToInsert.Count % ((dataTable.Columns.Count) - 1) == 0)
{
_observationRepository.InsertMany(obsReadyToInsert);
obsReadyToInsert.Clear();
}
}
}
}
if (obsReadyToInsert.Count > 0)
{
Debug.WriteLine("Created Observations are NOT Equal to measured values in the file please check!!");
}
dataFile.State = "LOADED";
dataFile.IsLoadedToDB = true;
_dataFileRepository.Update(dataFile);
_dataContext.Save();
return(true);
}
/*
* public void LoadObservations(Dataset dataset, SdtmRowDescriptor sdtmEntityDescriptor, List<SdtmRow> sdtmRowList)
* {
*
* //Retrieve all Observtion Descriptors for this domain and this project if previously a dataset of the same domain has been loaded?
* var obsDescriptionList = _ObsDescriptionRepository.FindAll(
* s => s.ProjectId.Equals(dataset.Activity.ProjectId) && s.DomainId.Equals(dataset.DomainId),
* new List<Expression<Func<ObservationDescriptor, object>>>() { od => od.O3 }).ToList();
*
* var O3List = new List<ObjectOfObservation>();
* if (obsDescriptionList.Count != 0)
* O3List.AddRange(obsDescriptionList.Select(observationDescription => observationDescription.O3));
*
#region Create O3 and ObservationDescription
* foreach (var sdtmRow in sdtmRowList)
* {
* //O3 name is not enough to uniquely identify the observation in place
* //e.g. white blood cells with two categories Heamatology and Chemistry
* //TODO: if an observation was previoulsy loaded via a dataset that had no group for it
* //e.g. white blood cells that was meant to be heamtology but not explicitly mentioned
* //then the same white blood cells is found but with a group, it will be loaded as a different O3
* //In any case, the CVterm of whitebloodcells for heamatology should be different from whitebloodcell chemsitry
* //ACTUALLY TOPIC variable SHOULD not be white blood cells but rather the TEST itself which in this case would be
* //Heamtatology test
* string O3 = sdtmRow.Topic; //THIS IS --TESTCD AND --TERM FOR NOW
* ObjectOfObservation O3obj;
* ObservationDescriptor obsDescriptor;
*
*
*
* if (sdtmRow.Group != null && sdtmRow.Subgroup != null)
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group)
* && o3.Subgroup.Equals(sdtmRow.Subgroup));
* else if (sdtmRow.Group != null)
* {
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group));
* }
* else
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode));
* if (O3obj != null) continue;
*
* //Create new O3
* O3obj = new ObjectOfObservation();
* O3obj.Name = sdtmRow.Topic; //EOS
* O3obj.Synonym = sdtmRow.TopicSynonym ?? null; //Eosinophils
* O3obj.ProjectId = dataset.Activity.ProjectId; //id for P-BVS
* O3obj.DomainId = dataset.DomainId; //id for D-SDTM-LB
* O3obj.CVTermStr = sdtmRow.TopicControlledTerm ?? null; //TEMP //Value of DECOD or LOINC
* //O3obj.CVtermId = lookupCVterm(sdtmRow.TopicVariableControlledTerm);
* O3obj.Group = sdtmRow.Group; //Hematology
* O3obj.Subgroup = sdtmRow.Subgroup; //value of SCAT
* _O3Repository.Insert(O3obj);
* O3List.Add(O3obj);
*
* //Create ObservationDescriptor
* obsDescriptor = new ObservationDescriptor();
* obsDescriptor.O3 = O3obj;
* obsDescriptor.DomainId = dataset.DomainId;
* obsDescriptor.ProjectId = dataset.Activity.ProjectId;
*
* _ObsDescriptionRepository.Insert(obsDescriptor);
*
* }
* _dataContext.Save();
#endregion
*
* obsDescriptionList = _ObsDescriptionRepository.FindAll(
* s => s.ProjectId.Equals(dataset.Activity.ProjectId) && s.DomainId.Equals(dataset.DomainId),
* new List<Expression<Func<ObservationDescriptor, object>>>() { od => od.O3 }).ToList();
*
* //NOW WITH IDS TO PASS ON TO MONGO
* O3List = new List<ObjectOfObservation>();
* O3List.AddRange(obsDescriptionList.Select(observationDescription => observationDescription.O3));
*
* //ALL descriptors mapped from the SDTM entity descriptor
* var descriptorsList = LoadObsDescriptors(sdtmEntityDescriptor, dataset.Activity.ProjectId);
*
* foreach (var sdtmRow in sdtmRowList)
* {
* //WARNING
* //depending ON THE STRING EQUALITY HERE TO IDENTIFY O3 ACROSS DATASETS AND ACROSS STUDIES OF THE SAME PROJECT
* //SHOULD REALLY USE THE CONTROLLED TERM INSTEAD (I.E. DECODE / LOINC)
*
* //1- GET O3
* ObjectOfObservation O3obj;
* string O3 = sdtmRow.Topic; //THIS IS --TESTCD AND --TERM FOR NOW
* if (sdtmRow.Group != null && sdtmRow.Subgroup != null)
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group)
* && o3.Subgroup.Equals(sdtmRow.Subgroup));
* else if (sdtmRow.Group != null)
* {
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group));
* }
* else
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode));
*
*
* //2- Get its observationDefinition
* var obsDescription = obsDescriptionList.Find(od => od.ObjectOfObservationId.Equals(O3obj.Id));
*
* //3- Create new observation
* Study study = _studyRepository.FindSingle(s => s.Name.Equals(sdtmRow.StudyId));
* var observation = new Core.Domain.Model.ObservationModel.Observation();
* observation.O3Id = O3obj.Id;
* observation.ObjectOfObservation = O3obj.Name;
* observation.Study = sdtmRow.StudyId;
* observation.StudyDBId = study != null ? study.Id : 0;
* observation.SO2Id = sdtmRow.USubjId;
* observation.SO2Type = "SUBJECT";
*
*
#region 3.1 Add Default property value
*
* //THE DEFAULT DESCRIPTOR IS NOT A NATIVE DEFAULT DESCRIPTOR PRE-SPECIFIED ,
* //IT'S just a regular descriptor which could take on any qualifier value
* //e.g. OCCUR, ORRES, STRESC, STRESN
* // and only at the time of creating the obsevation first time the observationDefinition.DefaultQualifier will be set to one of the qualifiers
* //based on ... Cannot be based on information pre-assigned to the SDTMentity descriptor since that applies to the dataset as a whole and not on a row
* //basis (will not solve the situation where some observations will have their defualt set to STRESN and others set to STRESC)
* // so at the time of observation creation first time based on the following rules set its observationDefinition.DefaultQualifier
* // if domain is findings and if there's a value for STRESN then this is the default,
* //else if there's a value for STRESC then this is default else ORRES is the default
* //this equality is going to be achieved
*
*
* VariableDefinition defQVariable = null; //the variable that will be read from the sdtm row as the default
* PropertyDescriptor defDescriptor; //= obsDescription.DefaultPropertyDescriptor; //the matching descriptor and set it to observationDescription
*
* if (dataset.Domain.Class.ToLower().Equals("findings"))
* {
* var defVar = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESN"));
* if (sdtmRow.ResultQualifiers["STRESN"] != null)
* {
* defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESN"));
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("STRES"));
* }
* else if (sdtmRow.ResultQualifiers["STRESC"] != null)
* {
* defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESC"));
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("STRES"));
* }
* else //if (sdtmRow.ResultQualifiers["ORRES"] != null)
* {
* defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("ORRES"));
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("ORRES"));
* }
* }
* else if (sdtmEntityDescriptor.DomainCode.Equals("AE"))
* {
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("OCCUR"));
* //there is no defQvariable in this case as we will make it up it doesn not occur in the original dataset
* }
* else
* {
* defQVariable = sdtmEntityDescriptor.DefaultQualifier;
* defDescriptor = descriptorsList.Find(d => d.Name.Equals(defQVariable.Name));
* }
*
* obsDescription.DefaultPropertyDescriptor = defDescriptor;
*
* //TODO: THIS COULD EITHER BE CVTERMS SO BOTH VARIABLE DEFINITION AND OBSERVATION DESCRIPTOR WOULD SHARE THE SAME CVTERM
* //todo: CHANGE ALL VARIABLE NAMES TO ENUMS
*
*
* if (sdtmEntityDescriptor.DomainCode.Equals("AE"))
* {
* CategoricalValue propValue = new CategoricalValue();
* propValue.Property = defDescriptor;
* propValue.Value = "Y";
* }
* else if (defDescriptor.ValueType == ObsValueType.Numerical)
* {
* NumericalValue propertyValue = new NumericalValue();
* propertyValue.Value = float.Parse(sdtmRow.ResultQualifiers[defQVariable.Name]);
* //propertyValue.Unit =
* propertyValue.Property = defDescriptor;
* observation.DefaultObservedProperty = propertyValue;
* }
* else
* {
* CategoricalValue propValue = new CategoricalValue();
* propValue.Property = defDescriptor;
* propValue.Value = sdtmRow.ResultQualifiers[defQVariable.Name];
* }
#endregion
*
* ///////////////////////////////////////////////////////////////////////////////////////////
*
#region 3.2 Add other qualifiers
*
* foreach (var qualifier in sdtmRow.Qualifiers)
* {
* var key = qualifier.Key.Substring(2);
* //GET THE MATCHING VARIABLE FROM the sdtm desriptor using names qualifier.key ==
* //then use its name
* if (qualifier.Value != null)
* {
* //ADD descriptor to obsDescription if not previously added
* //HOW IS THE LINK BETWEEN THE VALUE IN THE SDTM ENTITY AND THE DESCRIPTOR IS ESTABLISHED
* //CURRENTLY IT IS USING THE SAME NAME FOR THE DESCRIPTOR AS THE NAME OF THE VARIABLE WITHOUT THE PREFIX
*
* PropertyDescriptor descriptor = obsDescription.ObservedPropertyDescriptors.Find(d => d.Name.Equals(key));
* if (descriptor == null)
* {
* descriptor = descriptorsList.Find(d => d.Name.Equals(key));
* obsDescription.ObservedPropertyDescriptors.Add(descriptor);
* }
* //ADD new observation instance
*
* if(descriptor.ValueType.Equals(ObsValueType.Categorical))
* //TODO: for now not checking on property value type and creating categorical value for all other than result for findings
* observation.ObservedProperties.Add(new CategoricalValue()
* {
* Value = qualifier.Value,
* Property = descriptor,
*
* //CVTerm =
* });
*
*
*
* }
* }
*
#endregion
*
* ///////////////////////////////////////////////////////////////////////////////////////////
*
#region 4 Add Temporal Qualifiers
*
#endregion
*
#region 5 Add TimeSeries Descriptors
*
#endregion
* }
* }
*
* internal List<PropertyDescriptor> LoadObsDescriptors(SdtmRowDescriptor sdtmEntityDescriptor, int projectId)
* {
*
* var descriptorList = new List<PropertyDescriptor>();
*
* // SetDescriptorTypes(sdtmEntityDescriptor);
*
* //IF DOMAIN.CLASS which can be identified from sdtmEntityDescriptor
* //THEN cREATE descriptor for MATCHIN CLASS
* //BUT then how will this be linked with the variable name in the data file?
*
#region HACK
* //TODO: chage this to enum
*
* //A HACK TO ADD OCCURENCE descriptor for adverse events which would not be in the standard
* if (sdtmEntityDescriptor.DomainCode.Equals("AE"))
* {
* var descriptor = _ObsDescriptorRepository.FindSingle(d => d.Name.Equals("OCCUR"));
* //o => o.Type == DescriptorType.
* //&& o.Domain.Equals(sdtmEntityDescriptor.DomainCode));
*
* if (descriptor == null)
* {
* descriptor = PropertyDescriptorFactory.CreateAEdefDescriptor();
* _ObsDescriptorRepository.Insert(descriptor);
* }
* descriptorList.Add(descriptor);
* }
*
* //A HACK BECAUSE I DONT WANT TO HAVE TWO STANDARD RESULT DESCRIPTORS AS IT IS THE CASE WITH SDTM VARAIBLES
* if (sdtmEntityDescriptor.Class.Equals("Findings"))
* {
* var descriptorST = _ObsDescriptorRepository.FindSingle(d => d.Name.Equals("STRES"));
* if (descriptorST == null)
* {
* descriptorST = PropertyDescriptorFactory.CreateStandardResDescriptor();
* _ObsDescriptorRepository.Insert(descriptorST);
* }
* descriptorList.Add(descriptorST);
* }
#endregion
*
* foreach (var variable in sdtmEntityDescriptor.QualifierVariables.Union(sdtmEntityDescriptor.ResultVariables))
* {
* //todo: SHOULD BE ABLE TO DIFFERENTIATE HERE THE DIFFERENT SUBCLASSES OF SDTM RECORD QUALIFIERS
* //TODO: INTO FEATURE_QUALIFIERS, DEFAULT_OBSERVED_PROPERTY, OBSERVED_PROPERTIES, SUBJECT_QUALIFIERS AND SAMPLE_QUALIFIERS
* //TODO: should find equality via CVterm not name text
* //TODO: CVterm should be assigned to each variable definition which shuold then be an EQUIVALENT CVterm used for the descriptor
* var descriptor = _ObsDescriptorRepository.FindSingle(o => o.Name.Equals(variable.Name.Substring(2)));
* if (descriptor == null)
* {
* //FOR NOW the Descriptor is CREATED FROM the variable information
* descriptor = new PropertyDescriptor();
* descriptor.Name = variable.Name.Substring(2);
* //SOME KIND OF A METHOD HERE THAT RETURNS THE DESCRIPTOR TYPE THIS VARIABLE MATCHES
* //descriptor.Type = variable.DescriptorType; //THIS IS SUPPOSED TO HAVE NOW THE NEW SUBCLASSES //DescriptorType.ObservedPropertyDescriptor;
* descriptor.Description = variable.Label;
* descriptor.ProjectId = projectId;
*
* //_ObsDescriptorRepository.Insert(descriptor);
* }
* descriptorList.Add(descriptor);
* }
*
* //_dataContext.Save();
*
* //TODO: consider doing the same for other qualifiers such as timings and setting variable qualifiers to qualifiers
*
* return descriptorList;
* }
*/
internal bool LoadObservations(List <SdtmRow> sdtmData, SdtmRowDescriptor sdtmRowDescriptor, bool reload)
{
var dsDomainCode = sdtmRowDescriptor.DomainCode;
var dsClass = sdtmRowDescriptor.Class;
var datasetId = sdtmData.First().DatasetId;
var dataFileId = sdtmData.First().DatafileId;
var projectId = sdtmData.First().ProjectId;
if (reload)
{
_ObservationRepository.DeleteMany(o => o.DatasetId == datasetId && o.DatafileId == dataFileId);
_dataContext.Save().Equals("CREATED");
}
//Retrieve ObjectOfObservations previously loaded for this project
// CURRENTLY THIS IS NOT REALLY THE OBJECTOFOBSERVATION BUT THE OBSERVATION DESCRIPTOR WHICH IS
//DEFINED PER DATASET/DATAFILE AND IS DELETABLE ACROSS LOADS AS LONG AS THE O3 CV IS UNIQUE ACROSS DATASETS
//AND NOT JUST INFLUENCED BY ONE DATASET OR THE FIRST DATASET LOADED AS IT IS NOW!!
var projectO3s = _ObservationRepository.FindAll(o => o.ProjectId == projectId && o.DomainCode == dsDomainCode).ToList();
List <string> O3keys = projectO3s.Select(currObservation => currObservation.Class + currObservation.DomainCode + currObservation.Group + currObservation.Name).ToList();
var observations = sdtmData.GroupBy(o => new { domain = o.DomainCode, o3 = o.Topic, group = o.Group, subgroup = o.Subgroup, o3CVterm = o.TopicControlledTerm ?? o.TopicSynonym });
foreach (var observation in observations)
{
var O3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3;
if (O3keys.Contains(O3key))
{
continue;
}
Observation obsDescriptor = new Observation();
obsDescriptor.Name = observation.Key.o3;
obsDescriptor.Group = observation.Key.group;
obsDescriptor.Subgroup = observation.Key.subgroup;
obsDescriptor.Class = sdtmRowDescriptor.Class;
obsDescriptor.DomainCode = sdtmRowDescriptor.DomainCode;
obsDescriptor.DomainName = sdtmRowDescriptor.Domain;
obsDescriptor.TopicVariable = sdtmRowDescriptor.TopicVariable;
if (sdtmRowDescriptor.ObsIsAFinding)
{
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.ResultVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor, Qualifier = qualifier
}));
}
//sdtmRowDescriptor.ResultVariables.Union(sdtmRowDescriptor.QualifierVariables).ToList())};
else
{
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.QualifierVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
}
obsDescriptor.Timings.AddRange(sdtmRowDescriptor.GetAllTimingVariables().FindAll(v => v != null).Select(
qualifier => new ObservationTiming()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
obsDescriptor.ControlledTermStr = observation.Key.o3CVterm;
obsDescriptor.DatasetId = datasetId;
obsDescriptor.DatafileId = dataFileId;
obsDescriptor.ProjectId = projectId;
obsDescriptor.DefaultQualifier = sdtmRowDescriptor.GetDefaultQualifier(observation.FirstOrDefault());
// //LOAD CVTERMS CORRESPONDING TO MEDdRA codes
// ob = CreateObsCVterms(observation.FirstOrDefault(), sdtmRowDescriptor, out ob);
// ob.ControlledTermId = observation.FirstOrDefault().QualifierQualifiers?[AECVtermIdVar];
//}
_ObservationRepository.Insert(obsDescriptor);
}
var success = _dataContext.Save().Equals("CREATED");
//addO3IdstoSDTMrows()
if (success)
{
//TODO: PROBLEM!!
//IN CASE OF A SECOND FILE LOADED TO A PREVIOUSLY CREATED DATASET, DATAFILE WILL BE DIFFERENT
//OBSERVATIONS WERE SAVED WITH THE FISRT DATASETID AND THE FIRST DATAFILE
//WHAT HAPPENS WHERE WE WANT TO UNLOAD A FILE THAT BROUGHT DIFFERENT O3s to the DATASET DIFFERENT
//FROM THE FIRST DATAFILE?
//I STILL NEED TO BE ABLE TO FIND O3s that were loaded from a certain DATAFILE
var savedObs = _ObservationRepository.FindAll(o => o.DatasetId == datasetId && o.DatafileId == dataFileId).ToList();
foreach (var observation in observations)
{
foreach (var sdtmRow in observation)
{
var O3 =
savedObs.FirstOrDefault(
o => o.Name == sdtmRow.Topic && o.Group == sdtmRow.Group && o.Subgroup == sdtmRow.Subgroup);
if (O3 != null)
{
sdtmRow.DBTopicId = O3.Id;
}
_sdtmRepository.Update(sdtmRow);
}
}
}
return(success);
}
public async Task <bool> LoadSubjects(Dataset dataset, int fileId, bool reload)
{
var descriptor = SdtmSubjectDescriptor.GetSdtmSubjectDescriptor(dataset);
List <SdtmRow> sdtmData = await _sdtmRepository.FindAllAsync(
dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId));
if (sdtmData.Count == 0)
{
return(false);
}
var projectId = sdtmData.First().ProjectId;
var projectAccession = sdtmData.First().ProjectAccession;
var activityId = sdtmData.First().ActivityId;
var datafileId = sdtmData.First().DatafileId;
if (reload)
{
_subjectRepository.DeleteMany(o => o.DatasetId == dataset.Id && o.DatafileId == datafileId);
_dataContext.Save().Equals("CREATED");
}
//Project related subject characteristics
var scoList = _characteristicObjRepository.FindAll(s => s.ProjectId == projectId).ToList();
var scos = scoList.ToDictionary(co => co.ShortName);
//Project related studies
var studies = _studtRepository.FindAll(s => s.ProjectId == projectId, new List <string> {
"Arms.Arm"
}).ToList();
var studyMap = studies.ToDictionary(study => study.Name);
//Project related arms
var arms = studies.SelectMany(s => s.Arms.Select(a => a.Arm)).Distinct();
var armMap = arms.ToDictionary(arm => arm.Name);
//Previously loaded subjects for this project that might be update by this load
//TODO: should I add datafileId to subject as well? to be able to delete subjects for this datafile
var subjects = _subjectRepository.FindAll(s => s.Study.ProjectId == projectId).ToList();
foreach (var sdtmSubject in sdtmData)
{
Study study;
if (!studyMap.TryGetValue(sdtmSubject.StudyId, out study))
{
study = new Study()
{
Name = sdtmSubject.StudyId,
Accession = "S-" + projectAccession.Substring(2) + "-" + (studies.Count() + 1).ToString("00"),
ProjectId = projectId
}; studyMap.Add(study.Name, study);
}
Arm arm;
if (!armMap.TryGetValue(sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name], out arm))
{
arm = new Arm()
{
Id = "P-" + projectId + "-ARM-" + sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name],
Code = sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name],
Name = sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name]
};
armMap.Add(arm.Name, arm);
}
if (!arm.Studies.Exists(s => s.Study.Name == study.Name))
{
arm.Studies.Add(new StudyArm()
{
Arm = arm, Study = study
});
}
var subjNewFlag = false;
/**
* ADDING SUBJECTS
*/
var subject = subjects.Find(s => s.UniqueSubjectId == sdtmSubject.USubjId);
if (subject == null)
{
subject = new HumanSubject
{
Id = "P-" + projectId + "-" + sdtmSubject.USubjId,
UniqueSubjectId = sdtmSubject.USubjId,
SubjectStudyId = sdtmSubject.SubjectId,
Arm = sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name], //Will put in characteristics for now
ArmCode = sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name],
DatasetId = sdtmSubject.DatasetId,
DatafileId = sdtmSubject.DatafileId,
Study = study,
StudyArm = arm
};
subjNewFlag = true;
}
else
{
subjNewFlag = false;
subject.DatasetId = dataset.Id;
subject.DatafileId = sdtmSubject.DatafileId;
subject.Arm = sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name];
subject.ArmCode = sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name];
subject.Study = study;
subject.StudyArm = arm;
}
//SET/UPDATE SUBJECT CHARACTERISTICS
if (descriptor.RefStartDate != null && sdtmSubject.Qualifiers.ContainsKey(descriptor?.RefStartDate?.Name))
{
DateTime startDate;
if (DateTime.TryParse(sdtmSubject.Qualifiers[descriptor.RefStartDate?.Name], out startDate))
{
subject.SubjectStartDate = startDate;
}
}
if (descriptor.RefEndDate != null && sdtmSubject.Qualifiers.ContainsKey(descriptor?.RefEndDate?.Name))
{
DateTime endDate;
if (DateTime.TryParse(sdtmSubject.Qualifiers[descriptor.RefEndDate.Name], out endDate))
{
subject.SubjectEndDate = endDate;
}
}
//THIS METHOD WILL DELETE ALL SUBJECT CHARACTERISTICS PREVIOUSLY LOADED
//FOR THIS SUBJECT FOR THE SAME DATAFILE
//((List<SubjectCharacteristic>)subject.SubjectCharacteristics)
// .RemoveAll(sc=>sc.DatafileId == datafileId && sc.DatasetId == datasetId);
//_subjectRepository.Update(subject);
//_dataContext.Save();
/**
* ADDING SUBJECT CHARACTERISTICS
*/
foreach (var qualifier in sdtmSubject.Qualifiers)
{
//THIS CHECKS THAT ONLY COLUMNS IN DATA THAT ARE PRESENT IN THE TEMPLATE ARE PARSED
var dsVar = descriptor.QualifierVariables.SingleOrDefault(v => v.Name.Equals(qualifier.Key));
if (descriptor.CharacteristicProperties.Exists(c => c.Name == qualifier.Key) ||
descriptor.TimingVariableDefinitions.Exists(c => c.Name == qualifier.Key))
{
CharacteristicFeature sco;
scos.TryGetValue(qualifier.Key, out sco);
if (sco == null)
{
/**
* CREATING NEW CHARACTERISTIC OBJECT
*/
sco = new CharacteristicFeature()
{
ShortName = dsVar.Name,
FullName = dsVar.Label,
Domain = "DM",
ProjectId = projectId,
DataType = dsVar.DataType,
ActivityId = activityId
};
scos.Add(dsVar.Name, sco);
}
/**
* ADDING SUBJECT CHARACTERISTIC
*/
subject.SubjectCharacteristics.Add(new SubjectCharacteristic()
{
CharacteristicFeature = sco,
VerbatimValue = qualifier.Value,
VerbatimName = sco.ShortName,
Subject = subject,
DatasetId = dataset.Id,
DatafileId = datafileId
});
}
}
if (subjNewFlag)
{
_subjectRepository.Insert(subject);
}
else
{
_subjectRepository.Update(subject);
}
}
return(_dataContext.Save().Equals("CREATED"));
}
public bool LoadBioSamples(List <SdtmRow> sampleData, int datasetId)
{
var dataset = _datasetRepository.FindSingle(d => d.Id.Equals(datasetId), new List <string>()
{
"Variables.VariableDefinition"
});
var studyMap = new Dictionary <string, int>();
var scoList = _characteristicObjRepository.FindAll(s => s.ProjectId.Equals(dataset.Activity.ProjectId)).ToList();
var scos = scoList.ToDictionary(co => co.ShortName);
foreach (SdtmRow sdtmEntity in sampleData)
{
Study study;
int studyid;
if (!studyMap.TryGetValue(sdtmEntity.StudyId, out studyid))
{
study = _studyRepository.FindSingle(s => s.Name.Equals(sdtmEntity.StudyId));
studyMap.Add(sdtmEntity.StudyId, study.Id);
studyid = study.Id;
}
/**
* ADDING BIOSAMPLE
*/
var bioSample = new Biosample()
{
BiosampleStudyId = sdtmEntity.SampleId,
AssayId = sdtmEntity.ActivityId,
SubjectId = sdtmEntity.USubjId,
StudyId = studyid,
//Visit = sdtmEntity.Visit,
//CollectionStudyDay = sdtmEntity.CollectionStudyDay,
DatasetId = sdtmEntity.DatasetId,
IsBaseline = sdtmEntity.BaseLineFlag
};
/**
* ADDING SAMPLE CHARACTERISTICS
*/
foreach (var resqualifier in sdtmEntity.ResultQualifiers)
{
var dsVar = dataset.Variables.SingleOrDefault(v => v.VariableDefinition.Name.Equals(resqualifier.Key));
if (dsVar != null)
{
CharacteristicFeature sco;
scos.TryGetValue(sdtmEntity.Topic, out sco);
if (sco == null)
{
/**
* CREATING NEW CHARACTERISTIC OBJECT
*/
sco = new CharacteristicFeature()
{
ShortName = sdtmEntity.Topic, //characteristicVar.VariableDefinition.Name,
FullName = sdtmEntity.TopicSynonym, //characteristicVar.VariableDefinition.Label,
Domain = "BS",
ProjectId = dataset.Activity.ProjectId,
//DataType = dsVar.VariableDefinition.DataType
};
scos.Add(sdtmEntity.Topic, sco);
}
bioSample.SampleCharacteristics.Add(new SampleCharacteristic()
{
// DatasetVariable = dsVar,
CharacteristicFeature = sco,
VerbatimName = sdtmEntity.TopicSynonym,
VerbatimValue = resqualifier.Value,
//DatasetDomainCode = sdtmEntity.DomainCode
});
}
}
foreach (var qualifier in sdtmEntity.Qualifiers)
{
var dsVar = dataset.Variables.SingleOrDefault(v => v.VariableDefinition.Name.Equals(qualifier.Key));
if (dsVar != null)
{
CharacteristicFeature sco;
scos.TryGetValue(qualifier.Key, out sco);
if (sco == null)
{
/**
* CREATING NEW CHARACTERISTIC OBJECT
*/
sco = new CharacteristicFeature()
{
ShortName = dsVar.VariableDefinition.Name,
FullName = dsVar.VariableDefinition.Label,
Domain = "BS",
ProjectId = dataset.Activity.ProjectId,
//DataType = dsVar.VariableDefinition.DataType
};
scos.Add(dsVar.VariableDefinition.Name, sco);
}
bioSample.SampleCharacteristics.Add(new SampleCharacteristic()
{
VerbatimName = qualifier.Key,
VerbatimValue = qualifier.Value,
//DatasetDomainCode = sdtmEntity.DomainCode,
//DatasetVariable =dsVar,
CharacteristicFeature = sco,
});
}
}
_bioSampleRepository.Insert(bioSample);
}
return(_dataContext.Save().Equals("CREATED"));
}
private UserDataset CreateOrFindSubjectClinicalDataset(CombinedQuery query, string userId)
{
var phenoDataset = _userDatasetRepository.FindSingle(
d => d.ProjectId == query.ProjectId &&
d.QueryId == query.Id &&
d.Type == "PHENO" &&
d.OwnerId == userId);
if (phenoDataset != null)
{
return(phenoDataset);
}
phenoDataset = new UserDataset
{
Id = Guid.NewGuid(),
OwnerId = userId,
ProjectId = query.ProjectId,
Type = "PHENO",
Name = "Subjects",
QueryId = query.Id,
FileStatus = 0,
};
//ADD SUBJECTID & STUDYID DATAFIELD
phenoDataset.Fields.Add(CreateSubjectIdField());
phenoDataset.Fields.Add(CreateStudyIdField());
//ADD DESIGN ELEMENT FIELDS (STUDY, VISIT, ARM...etc)
phenoDataset.Fields.AddRange(query.DesignElements.Select(qObj => new DatasetField()
{
QueryObject = qObj,
QueryObjectType = qObj.QueryFor,
ColumnHeader = qObj.QueryObjectName
}));
//ADD SUBJECT CHARACTERISTICS (AGE, RACE, SEX ...etc)
phenoDataset.Fields.AddRange(query.SubjectCharacteristics.Select(qObj => new DatasetField()
{
QueryObject = qObj,
QueryObjectType = nameof(SubjectCharacteristic),
ColumnHeader = qObj.QueryObjectName//.ObservationName
}));
//ADD CLINICAL OBSERVATIONS
foreach (var co in query.ClinicalObservations)
{
phenoDataset.Fields.Add(new DatasetField()
{
QueryObject = co,
QueryObjectType = nameof(SdtmRow),
ColumnHeader = co.ObservationName
});
if (co.HasLongitudinalData)
{
phenoDataset.Fields.Add(new DatasetField()
{
QueryObject = new Query()
{
QueryFor = nameof(Visit), QueryFrom = nameof(SdtmRow), QuerySelectProperty = "Name"
},
QueryObjectType = nameof(Visit),
ColumnHeader = "visit"
});
}
if (co.HasTPT)
{
phenoDataset.Fields.Add(new DatasetField()
{
QueryObject = new Query()
{
QueryFor = nameof(SdtmRow.CollectionStudyTimePoint), QueryFrom = nameof(SdtmRow), QuerySelectProperty = "Name"
},
QueryObjectType = nameof(RelativeTimePoint),
ColumnHeader = "timepoint"
});
}
}
//phenoDataset.Fields.AddRange(query.ClinicalObservations.Select(qObj => new DatasetField()
//{
// QueryObject = qObj,
// QueryObjectType = nameof(SdtmRow),
// ColumnHeader = qObj.ObservationName
//}));
//ADD GROUPED CLINICAL OBSERVATIONS
phenoDataset.Fields.AddRange(query.GroupedObservations.Select(gObs => new DatasetField()
{
QueryObject = gObs,
QueryObjectType = nameof(SdtmRow),
ColumnHeader = gObs.ObservationName
}));
var exportData = _queryService.GetQueryResult(query.Id);
phenoDataset.SubjectCount = exportData.Subjects.Count;
_userDatasetRepository.Insert(phenoDataset);
_dataContext.Save();
return(phenoDataset);
}
public async Task <bool> LoadObservations(Dataset dataset, int fileId, bool reload)
{
var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(dataset);
List <SdtmRow> sdtmData = await _sdtmRepository.FindAllAsync(
dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId));
var dsDomainCode = sdtmRowDescriptor.DomainCode;
var dsClass = sdtmRowDescriptor.Class;
var datasetId = sdtmData.First().DatasetId;
var dataFileId = sdtmData.First().DatafileId;
var projectId = sdtmData.First().ProjectId;
if (reload)
{
_observationRepository.DeleteMany(o => o.DatasetId == datasetId && o.DatafileId == dataFileId);
_dataContext.Save();
}
//Retrieve ObjectOfObservations previously loaded for this project
// CURRENTLY THIS IS NOT REALLY THE OBJECTOFOBSERVATION BUT THE OBSERVATION DESCRIPTOR WHICH IS
//DEFINED PER DATASET/DATAFILE AND IS DELETABLE ACROSS LOADS AS LONG AS THE O3 CV IS UNIQUE ACROSS DATASETS
//AND NOT JUST INFLUENCED BY ONE DATASET OR THE FIRST DATASET LOADED AS IT IS NOW!!
var projectO3s = _observationRepository.FindAll(o => o.ProjectId == projectId && o.DomainCode == dsDomainCode).ToList();
//List<string> O3keys =
// projectO3s.Select(
// currObservation =>
// currObservation.Class + currObservation.DomainCode + currObservation.Group +
// currObservation.Name).ToList();
var O3map = projectO3s.ToDictionary(o3 => o3.Class + o3.DomainCode + o3.Group + o3.Name + o3.ControlledTermStr);
var observations =
sdtmData.GroupBy(
o =>
new
{
domain = o.DomainCode,
o3 = o.Topic,
group = o.Group,
subgroup = o.Subgroup,
o3CVterm = o.TopicControlledTerm ?? o.TopicSynonym
});
var obsPrevLoaded = new Dictionary <string, Observation>();
foreach (var observation in observations)
{
var O3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3 + observation.Key.o3CVterm;
Observation o3;
if (O3map.TryGetValue(O3key, out o3))
{
obsPrevLoaded.Add(O3key, o3);
continue;
}
Observation obsDescriptor = new Observation();
obsDescriptor.Name = observation.Key.o3;
obsDescriptor.Group = observation.Key.group;
obsDescriptor.Subgroup = observation.Key.subgroup;
obsDescriptor.Class = sdtmRowDescriptor.Class;
obsDescriptor.DomainCode = sdtmRowDescriptor.DomainCode;
obsDescriptor.DomainName = sdtmRowDescriptor.Domain;
obsDescriptor.TopicVariable = sdtmRowDescriptor.TopicVariable;
if (sdtmRowDescriptor.ObsIsAFinding)
{
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.ResultVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
}
//sdtmRowDescriptor.ResultVariables.Union(sdtmRowDescriptor.QualifierVariables).ToList())};
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.QualifierVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
obsDescriptor.Timings.AddRange(sdtmRowDescriptor.TimeVariables.Select(
qualifier => new ObservationTiming()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
obsDescriptor.ControlledTermStr = observation.Key.o3CVterm;
obsDescriptor.DatasetId = datasetId;
obsDescriptor.DatafileId = dataFileId;
obsDescriptor.ProjectId = projectId;
obsDescriptor.DefaultQualifier = sdtmRowDescriptor.GetDefaultQualifier(observation.FirstOrDefault());
var newObs = _observationRepository.Insert(obsDescriptor);
obsPrevLoaded.Add(O3key, newObs);
}
var success = _dataContext.Save().Equals("CREATED");
if (success)
{
//TODO: PROBLEM!!
//IN CASE OF A SECOND FILE LOADED TO A PREVIOUSLY CREATED DATASET, DATAFILE WILL BE DIFFERENT
//OBSERVATIONS WERE SAVED WITH THE FISRT DATASETID AND THE FIRST DATAFILE
//WHAT HAPPENS WHERE WE WANT TO UNLOAD A FILE THAT BROUGHT DIFFERENT O3s to the DATASET DIFFERENT
//FROM THE FIRST DATAFILE?
//I STILL NEED TO BE ABLE TO FIND O3s that were loaded from a certain DATAFILE
var savedObs = _observationRepository.FindAll(o => o.DatasetId == datasetId && o.DatafileId == dataFileId).ToList();
foreach (var observation in observations)
{
foreach (var sdtmRow in observation)
{
Observation O3;
var o3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3 + observation.Key.o3CVterm;
if (obsPrevLoaded.TryGetValue(o3key, out O3))
{
sdtmRow.DBTopicId = O3.Id;
}
_sdtmRepository.Update(sdtmRow);
}
}
}
return(success);
}
public async Task <bool> LoadDataset(int datasetId, int fileId)
{
Dataset dataset = GetDatasetDescriptor(datasetId);
var reload = false;
var loaded = false;
var dataFile = _dataFileRepository.Get(fileId);
if (dataFile.IsLoadedToDB)
{
reload = true;
}
try
{
switch (dataset.Template.Code)
{
case "DM":
var subjectLoader = new SubjectLoader(_dataServiceUnit);
loaded = await subjectLoader.LoadSubjects(dataset, fileId, reload);
break;
case "BS":
var sampleLoader = new BioSampleLoader(_dataServiceUnit);
loaded = await sampleLoader.LoadBioSamples(dataset, fileId, reload);
break;
case "CY":
case "HD":
var hdDataLoader = new HDloader(_dataServiceUnit);
//loaded = await hdDataLoader.LoadHDdata(dataset,fileId, datasetId);
break;
case "MX":
break;
default:
var observationLoader = new ObservationLoader(this._dataServiceUnit);
loaded = await observationLoader.LoadObservations(dataset, fileId, reload);
if (loaded)
{
await _cacheService.GenerateAndCacheClinicalDTO(dataFile.ProjectId);
}
break;
}
if (loaded)
{
dataFile.State = "LOADED";
dataFile.IsLoadedToDB = true;
_dataFileRepository.Update(dataFile);
if (!dataset.DataFiles.Select(d => d.DatafileId).Contains(fileId))
{
//Adding a new datafile to this dataset
dataset.DataFiles.Add(new DatasetDatafile()
{
DatasetId = datasetId, DatafileId = dataFile.Id
});
_datasetRepository.Update(dataset);
}
_dataServiceUnit.Save();
}
if (!loaded && dataFile.State == "SAVED")
{
_sdtmRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId);
}
}
catch (Exception e)
{
Debug.WriteLine("Failed to load descriptors to SQL database", e.Message);
await UnloadDataset(datasetId, fileId, "LOADING FAILED");
}
return(loaded);
}
public async Task <bool> LoadBioSamples(Dataset dataset, int fileId, bool reload)
{
var descriptor = SdtmSampleDescriptor.GetSdtmSampleDescriptor(dataset);
List <SdtmRow> sampleData = await _sdtmRepository.FindAllAsync(
dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId));
if (sampleData.Count == 0)
{
return(false);
}
var projectId = sampleData.First().ProjectId;
var projectAccession = sampleData.First().ProjectAccession;
var assayId = sampleData.First().ActivityId;
var datasetId = sampleData.First().DatasetId;
var datafileId = sampleData.First().DatafileId;
if (projectId == 0 || assayId == 0)
{
return(false);
}
var assay = _assayRepository.Get(assayId);
if (reload)
{
_bioSampleRepository.DeleteMany(o => o.DatasetId == datasetId && o.DataFileId == datafileId);
_dataContext.Save().Equals("CREATED");
}
var studyMap = new Dictionary <string, int>();
//RETRIEVE PREVIOUSLY LOADED CHARACTERISTIC FEATURES FOR THIS ASSAY THAT MIGHT HAVE BEEN LOADED BEFORE FOR OTHER STUDIES
var featureList = _characteristicObjRepository.FindAll(s => s.ProjectId == projectId && s.ActivityId == assayId).ToList();
_featureMap = featureList.ToDictionary(co => co.ShortName);
var subjects = _subjectRepository.FindAll(s => s.Study.ProjectId == projectId).ToList();
var sampleDayMap = new Dictionary <string, int>();
foreach (var sdtmRow in sampleData)
{
//RETRIEVE STUDYID FROM DB AND STORE IN MAP LOCALLY TO AVOID MAKING TOO MANY DB CALLS
int studyid;
if (!studyMap.TryGetValue(sdtmRow.StudyId, out studyid))
{
var study = _studyRepository.FindSingle(s => s.Name == sdtmRow.StudyId && s.ProjectId == projectId);
studyMap.Add(sdtmRow.StudyId, study.Id);
studyid = study.Id;
}
/**
* ADDING BIOSAMPLE
*/
var bioSample = new Biosample()
{
BiosampleStudyId = sdtmRow.SampleId,
AssayId = sdtmRow.ActivityId,
SubjectId = subjects.Find(s => s.UniqueSubjectId == sdtmRow.USubjId)?.Id,
StudyId = studyid,
CollectionStudyDay = sdtmRow.CollectionStudyDay,
DatasetId = sdtmRow.DatasetId,
DataFileId = sdtmRow.DatafileId,
IsBaseline = sdtmRow.Qualifiers != null && descriptor.IsBaseLineVariable != null && (sdtmRow.Qualifiers.ContainsKey(descriptor.IsBaseLineVariable?.Name) && sdtmRow.Qualifiers[descriptor.IsBaseLineVariable?.Name] == "Y")
};
//CHECK IF ASSAY HAS TEMOPORALDATA
if (bioSample.CollectionStudyDay?.Number != null && !assay.HasTemporalData)
{
if (sampleDayMap.ContainsKey(bioSample.SubjectId) &&
sampleDayMap[bioSample.SubjectId] != bioSample.CollectionStudyDay.Number.Value &&
!assay.HasTemporalData)
{
assay.HasTemporalData = true;
}
else
{
if (!sampleDayMap.ContainsKey(bioSample.SubjectId))
{
sampleDayMap.Add(bioSample.SubjectId, bioSample.CollectionStudyDay.Number.Value);
}
}
}
//ADD CHARACTERISTIC FEATURE IN BSTEST
string charValue;
var characValueVariable = descriptor.GetValueVariable(sdtmRow);
if (characValueVariable != null && sdtmRow.ResultQualifiers.TryGetValue(characValueVariable.Name, out charValue))
{
var charFeature = getCharacteristicFeature(sdtmRow.Topic, sdtmRow.TopicSynonym, characValueVariable.DataType);
charFeature.ActivityId = assayId;
charFeature.ProjectId = projectId;
bioSample.SampleCharacteristics.Add(new SampleCharacteristic()
{
CharacteristicFeature = charFeature,
VerbatimName = sdtmRow.TopicSynonym,
VerbatimValue = charValue,
DatafileId = datafileId,
DatasetId = datasetId
});
}
//ADDING SPECIMEN TYPE TO CHARACTERISTICS
string st;
if (descriptor.SpecimenTypeVariable != null && sdtmRow.Qualifiers.TryGetValue(descriptor.SpecimenTypeVariable.Name, out st))
{
var feature = getCharacteristicFeature(descriptor.SpecimenTypeVariable.Name, descriptor.SpecimenTypeVariable.Label,
descriptor.SpecimenTypeVariable.DataType);
feature.ActivityId = assayId;
feature.ProjectId = projectId;
bioSample.SampleCharacteristics.Add(new SampleCharacteristic()
{
CharacteristicFeature = feature,
VerbatimName = descriptor.SpecimenTypeVariable?.Name,
VerbatimValue = st,
DatafileId = datafileId,
DatasetId = datasetId
});
}
//ADD ANATOMIC REGION TO CHARACTERISTICS
if (descriptor.AnatomicRegionVariable != null && sdtmRow.QualifierQualifiers.TryGetValue(descriptor.AnatomicRegionVariable.Name, out st))
{
var feature = getCharacteristicFeature(descriptor.AnatomicRegionVariable.Name, descriptor.AnatomicRegionVariable.Label,
descriptor.AnatomicRegionVariable.DataType);
feature.ActivityId = assayId;
feature.ProjectId = projectId;
bioSample.SampleCharacteristics.Add(new SampleCharacteristic()
{
CharacteristicFeature = feature,
VerbatimName = descriptor.AnatomicRegionVariable.Name,
VerbatimValue = st,
DatafileId = datafileId,
DatasetId = datasetId
});
}
_bioSampleRepository.Insert(bioSample);
_assayRepository.Update(assay);
}
return(_dataContext.Save().Equals("CREATED"));
}
public bool LoadSDTM(int datasetId, int fileId, DataTable dataTable)
{
var dataset = _datasetRepository.FindSingle(d => d.Id == datasetId, new List <string>()
{
"Variables.VariableDefinition", "Template", "Activity.Project"
});
var dataFile = _dataFileRepository.Get(fileId);
if (!dataFile.State.ToLower().Equals("new"))
{
/**
* Replacing previously loaded file
* Remove file from collection before reloading it
*/
_sdtmRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId);
Debug.WriteLine("RECORD(s) SUCCESSFULLY DELETED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId);
}
var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(dataset);
var SDTM = new List <SdtmRow>();
var totalLoaded = 0.0;
try
{
foreach (DataRow row in dataTable.Rows)
{
var sdtmRow = SDTMreader.readSDTMrow(row, dataTable, sdtmRowDescriptor);
sdtmRow.Id = Guid.NewGuid();
sdtmRow.DatasetId = datasetId;
sdtmRow.ActivityId = dataset.ActivityId;
sdtmRow.DatafileId = fileId;
sdtmRow.ProjectId = dataset.Activity.ProjectId;
sdtmRow.ProjectAccession = dataset.Activity.Project.Accession;
SDTM.Add(sdtmRow);
if (SDTM.Count % 100 == 0)
{
_sdtmRepository.InsertMany(SDTM);
totalLoaded += SDTM.Count;
dataFile.State = Math.Round(totalLoaded / dataTable.Rows.Count * 100).ToString("##");
_dataFileRepository.Update(dataFile);
_dataServiceUnit.Save();
SDTM.Clear();
}
}
_sdtmRepository.InsertMany(SDTM);
Debug.WriteLine(dataTable.Rows.Count + " RECORD(s) SUCCESSFULLY ADDED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId);
}
catch (Exception e)
{
dataFile.State = "FAILED";
dataFile.IsLoadedToDB = false;
_dataFileRepository.Update(dataFile);
_dataServiceUnit.Save();
Debug.WriteLine(e.Message);
return(false);
}
dataFile.State = "SAVED";
_dataFileRepository.Update(dataFile);
_dataServiceUnit.Save();
return(true);
}
