private Observation CreateObsCVterms(SdtmRow sdtmRow, SdtmRowDescriptor sdtmDescriptor, out Observation ob)
{
//const string AESOCvar = "AESOC";
//const string AEHG
throw new NotImplementedException();
}
public static List <SdtmRow> ReadSDTM(DataTable dataTable, SdtmRowDescriptor descriptor)
{
var SDTM = new List <SdtmRow>();
foreach (DataRow row in dataTable.Rows)
{
SDTM.Add(readSDTMrow(row, dataTable, descriptor));
}
return(SDTM);
}
public async Task <DataTable> ConsolidateDataset(int datasetId)
{
var sdtmRows = _sdtmRepository.FindAll(s => s.DatasetId == datasetId).OrderBy(s => s.StudyId);
var descriptor = GetDatasetDescriptor(datasetId);
var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(descriptor);
var dataTable = new DataTable();
if (descriptor.Activity.GetType().Equals(typeof(Assay)))
{
dataTable.TableName = descriptor.Activity.Name + "-" + descriptor.Template.Domain;
}
else
{
dataTable.TableName = descriptor.Template.Domain;
}
await Task.Run(() => {
foreach (var field in descriptor.Variables.OrderBy(vr => vr.OrderNumber).Select(v => v.VariableDefinition))
{
dataTable.Columns.Add(field.Name, typeof(string));
}
try
{
foreach (var sdtmrow in sdtmRows)
{
if (sdtmrow.DomainCode == null)
{
continue;
}
var row = SDTMreader.WriteSDTMrow(sdtmrow, dataTable, sdtmRowDescriptor);
dataTable.Rows.Add(row);
}
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
}
});
return(dataTable);
}
public static DataRow WriteSDTMrow(SdtmRow sdtmrow, DataTable datatable, SdtmRowDescriptor descriptor)
{
var row = datatable.NewRow();
foreach (DataColumn column in datatable.Columns)
{
var colName = column.ColumnName;
//var value = tableRow[column].ToString();
if (colName == descriptor.StudyIdentifierVariable.Name)
{
//"STUDYID":
row[column] = sdtmrow.StudyId ?? "";
}
else if (colName == descriptor.DomainVariable.Name)
{
//DOMAIN
row[column] = sdtmrow.DomainCode ?? "";
}
else if (colName == descriptor.SubjIdVariable?.Name)
{
//SUBJID
row[column] = sdtmrow.SubjectId ?? "";
}
else if (colName == descriptor.UniqueSubjIdVariable.Name)
{
//USUBJID
row[column] = sdtmrow.USubjId ?? "";
}
else if (colName == descriptor.SampleIdVariable?.Name)
{
//--REFID
row[column] = sdtmrow.SampleId ?? "";
}
else if (colName == descriptor.TopicVariable?.Name)
{
//--TESTCD // --TERM // --TRT
row[column] = sdtmrow.Topic ?? "";
}
else if (colName == descriptor.TopicSynonymVariable?.Name)
{
//--TEST // --MODIFY
row[column] = sdtmrow.TopicSynonym ?? "";
}
else if (colName == descriptor.TopicCVtermVariable?.Name)
{
//--LOINC --DECODE
row[column] = sdtmrow.TopicControlledTerm ?? "";
}
else if (colName == descriptor.GroupVariable?.Name)
{
//--CAT
row[column] = sdtmrow.Group ?? "";
}
else if (colName == descriptor.SubgroupVariable?.Name)
{
//--SCAT
row[column] = sdtmrow.Subgroup ?? "";
}
else if (descriptor.QualifierVariables.Select(v => v.Name).Contains(colName))
{
//RECORD QUALIFIERS
var val = sdtmrow.Qualifiers.FirstOrDefault(c => c.Key == colName).Value;
row[column] = val ?? "";
}
else if (descriptor.ResultVariables.Select(v => v.Name).Contains(colName))
{
//RESULT QUALIFIERS
var val = sdtmrow.ResultQualifiers.FirstOrDefault(c => c.Key == colName).Value;
row[column] = val ?? "";
}
else if (descriptor.SynonymVariables.Select(v => v.Name).Contains(colName))
{
//SYNOYMS
var val = sdtmrow.QualifierSynonyms.FirstOrDefault(c => c.Key == colName).Value;
row[column] = val ?? "";
}
else if (descriptor.VariableQualifierVariables.Select(v => v.Name).Contains(colName))
{
//VARIABLE QUALIFIERS
var val = sdtmrow.QualifierQualifiers.FirstOrDefault(c => c.Key == colName).Value;
row[column] = val ?? "";
}
else if (descriptor.TimeVariables.Select(v => v.Name).Contains(colName))
{
//VISIT
var val = sdtmrow.TimingQualifiers.FirstOrDefault(c => c.Key == colName).Value;
row[column] = val ?? "";
}
else if (descriptor.DurationVariable?.Name == colName)
{
//--DUR
row[column] = sdtmrow.Duration;
}
else if (sdtmrow.Leftovers.ContainsKey(colName))
{
var val = sdtmrow.Leftovers.FirstOrDefault(c => c.Key == colName).Value;
row[column] = val ?? "";
}
else
{
row[column] = "";
}
//TODO: else if(column is not mandatory add it and set null)
//else throw new Exception("Unknown column. Cannot find value for it.");column in descriptor
}
return(row);
}
public static SdtmRow readSDTMrow(DataRow tableRow, DataTable dataTable, SdtmRowDescriptor descriptor)
{
var sdtmrow = new SdtmRow();
sdtmrow.Class = descriptor.Class;
foreach (DataColumn column in dataTable.Columns)
{
var colName = column.ColumnName;
var value = tableRow[column].ToString();
if (colName == descriptor.StudyIdentifierVariable.Name)
{
//"STUDYID":
sdtmrow.StudyId = value;
}
else if (colName == descriptor.DomainVariable.Name)
{
//DOMAIN
sdtmrow.DomainCode = value;
}
else if (colName == descriptor.SubjIdVariable?.Name)
{
//SUBJID
sdtmrow.SubjectId = value;
}
else if (colName == descriptor.UniqueSubjIdVariable.Name)
{
//USUBJID
sdtmrow.USubjId = value;
}
else if (colName == descriptor.SampleIdVariable?.Name)
{
//--REFID
sdtmrow.SampleId = value;
}
else if (colName == descriptor.TopicVariable?.Name)
{
//--TESTCD // --TERM // --TRT
sdtmrow.Topic = value;
}
else if (colName == descriptor.TopicSynonymVariable?.Name)
{
//--TEST // --MODIFY
sdtmrow.TopicSynonym = value;
}
else if (colName == descriptor.TopicCVtermVariable?.Name)
{
//--LOINC --DECODE
sdtmrow.TopicControlledTerm = value;
}
else if (colName == descriptor.GroupVariable?.Name)
{
//--CAT
sdtmrow.Group = value;
}
else if (colName == descriptor.SubgroupVariable?.Name)
{
//--SCAT
sdtmrow.Subgroup = value;
}
else if (descriptor.QualifierVariables.Select(v => v.Name).Contains(colName))
{
//RECORD QUALIFIERS
sdtmrow.Qualifiers.Add(colName, value);
}
else if (descriptor.ResultVariables.Select(v => v.Name).Contains(colName))
{
//RESULT QUALIFIERS
sdtmrow.ResultQualifiers.Add(colName, value);
}
else if (descriptor.SynonymVariables.Select(v => v.Name).Contains(colName))
{
//SYNOYMS
sdtmrow.QualifierSynonyms.Add(colName, value);
}
else if (descriptor.VariableQualifierVariables.Select(v => v.Name).Contains(colName))
{
//VARIABLE QUALIFIERS
sdtmrow.QualifierQualifiers.Add(colName, value);
}
else if (descriptor.VisitNameVariable?.Name == colName)
{
//VISIT
sdtmrow.VisitName = value;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.VisitNumVariable?.Name == colName)
{
//VISITNUM
int num;
if (!int.TryParse(value, out num))
{
continue;
}
sdtmrow.VisitNum = int.Parse(value);
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.VisitPlannedStudyDay?.Name == colName)
{
//VISITNUM
int num;
if (!int.TryParse(value, out num))
{
continue;
}
sdtmrow.VisitPlannedStudyDay = int.Parse(value);
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.DateTimeVariable?.Name == colName)
{
//DTC
DateTime DT;
if (!DateTime.TryParse(value, out DT))
{
continue;
}
if (sdtmrow.CollectionDateTime == null)
{
sdtmrow.CollectionDateTime = new AbsoluteTimePoint();
}
sdtmrow.CollectionDateTime.DateTime = DT;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.StudyDayVariable?.Name == colName)
{
//--DY
int num;
if (!int.TryParse(value, out num))
{
continue;
}
if (sdtmrow.CollectionStudyDay == null)
{
sdtmrow.CollectionStudyDay = new RelativeTimePoint();
}
sdtmrow.CollectionStudyDay.Number = num;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.TimePointNameVariable?.Name == colName)
{
//--TPT
if (sdtmrow.CollectionStudyTimePoint == null)
{
sdtmrow.CollectionStudyTimePoint = new RelativeTimePoint();
}
sdtmrow.CollectionStudyTimePoint.Name = value;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.TimePointNameVariable?.Name == colName)
{
//--TPTNUM
int num;
if (!int.TryParse(value, out num))
{
continue;
}
if (sdtmrow.CollectionStudyTimePoint == null)
{
sdtmrow.CollectionStudyTimePoint = new RelativeTimePoint();
}
sdtmrow.CollectionStudyTimePoint.Number = num;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.DurationVariable?.Name == colName)
{
//--DUR
sdtmrow.Duration = value;
}
else if (descriptor.StartDateTimeVariable?.Name == colName)
{
//--STDTC
DateTime dt;
if (!DateTime.TryParse(value, out dt))
{
continue;
}
if (sdtmrow.DateTimeInterval == null)
{
sdtmrow.DateTimeInterval = new TimeInterval();
}
sdtmrow.DateTimeInterval.Start = new AbsoluteTimePoint();
((AbsoluteTimePoint)sdtmrow.DateTimeInterval.Start).DateTime = dt;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.EndDateTimeVariable?.Name == colName)
{
//--ENDTC
DateTime dt;
if (!DateTime.TryParse(value, out dt))
{
continue;
}
if (sdtmrow.DateTimeInterval == null)
{
sdtmrow.DateTimeInterval = new TimeInterval();
}
sdtmrow.DateTimeInterval.End = new AbsoluteTimePoint();
((AbsoluteTimePoint)sdtmrow.DateTimeInterval.End).DateTime = dt;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.StartStudyDayVariable?.Name == colName)
{
//--STDY
int val;
if (!int.TryParse(value, out val))
{
continue;
}
if (sdtmrow.StudyDayInterval == null)
{
sdtmrow.StudyDayInterval = new TimeInterval();
}
sdtmrow.StudyDayInterval.Start = new RelativeTimePoint();
((RelativeTimePoint)sdtmrow.StudyDayInterval.Start).Number = val;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else if (descriptor.EndStudyDayVariable?.Name == colName)
{
//--ENDY
int val;
if (!int.TryParse(value, out val))
{
continue;
}
if (sdtmrow.StudyDayInterval == null)
{
sdtmrow.StudyDayInterval = new TimeInterval();
}
sdtmrow.StudyDayInterval.End = new RelativeTimePoint();
((RelativeTimePoint)sdtmrow.StudyDayInterval.End).Number = val;
sdtmrow.TimingQualifiers.Add(colName, value);
}
else
{
sdtmrow.Leftovers.Add(colName, value);
}
}
return(sdtmrow);
}
/*
* public void LoadObservations(Dataset dataset, SdtmRowDescriptor sdtmEntityDescriptor, List<SdtmRow> sdtmRowList)
* {
*
* //Retrieve all Observtion Descriptors for this domain and this project if previously a dataset of the same domain has been loaded?
* var obsDescriptionList = _ObsDescriptionRepository.FindAll(
* s => s.ProjectId.Equals(dataset.Activity.ProjectId) && s.DomainId.Equals(dataset.DomainId),
* new List<Expression<Func<ObservationDescriptor, object>>>() { od => od.O3 }).ToList();
*
* var O3List = new List<ObjectOfObservation>();
* if (obsDescriptionList.Count != 0)
* O3List.AddRange(obsDescriptionList.Select(observationDescription => observationDescription.O3));
*
#region Create O3 and ObservationDescription
* foreach (var sdtmRow in sdtmRowList)
* {
* //O3 name is not enough to uniquely identify the observation in place
* //e.g. white blood cells with two categories Heamatology and Chemistry
* //TODO: if an observation was previoulsy loaded via a dataset that had no group for it
* //e.g. white blood cells that was meant to be heamtology but not explicitly mentioned
* //then the same white blood cells is found but with a group, it will be loaded as a different O3
* //In any case, the CVterm of whitebloodcells for heamatology should be different from whitebloodcell chemsitry
* //ACTUALLY TOPIC variable SHOULD not be white blood cells but rather the TEST itself which in this case would be
* //Heamtatology test
* string O3 = sdtmRow.Topic; //THIS IS --TESTCD AND --TERM FOR NOW
* ObjectOfObservation O3obj;
* ObservationDescriptor obsDescriptor;
*
*
*
* if (sdtmRow.Group != null && sdtmRow.Subgroup != null)
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group)
* && o3.Subgroup.Equals(sdtmRow.Subgroup));
* else if (sdtmRow.Group != null)
* {
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group));
* }
* else
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode));
* if (O3obj != null) continue;
*
* //Create new O3
* O3obj = new ObjectOfObservation();
* O3obj.Name = sdtmRow.Topic; //EOS
* O3obj.Synonym = sdtmRow.TopicSynonym ?? null; //Eosinophils
* O3obj.ProjectId = dataset.Activity.ProjectId; //id for P-BVS
* O3obj.DomainId = dataset.DomainId; //id for D-SDTM-LB
* O3obj.CVTermStr = sdtmRow.TopicControlledTerm ?? null; //TEMP //Value of DECOD or LOINC
* //O3obj.CVtermId = lookupCVterm(sdtmRow.TopicVariableControlledTerm);
* O3obj.Group = sdtmRow.Group; //Hematology
* O3obj.Subgroup = sdtmRow.Subgroup; //value of SCAT
* _O3Repository.Insert(O3obj);
* O3List.Add(O3obj);
*
* //Create ObservationDescriptor
* obsDescriptor = new ObservationDescriptor();
* obsDescriptor.O3 = O3obj;
* obsDescriptor.DomainId = dataset.DomainId;
* obsDescriptor.ProjectId = dataset.Activity.ProjectId;
*
* _ObsDescriptionRepository.Insert(obsDescriptor);
*
* }
* _dataContext.Save();
#endregion
*
* obsDescriptionList = _ObsDescriptionRepository.FindAll(
* s => s.ProjectId.Equals(dataset.Activity.ProjectId) && s.DomainId.Equals(dataset.DomainId),
* new List<Expression<Func<ObservationDescriptor, object>>>() { od => od.O3 }).ToList();
*
* //NOW WITH IDS TO PASS ON TO MONGO
* O3List = new List<ObjectOfObservation>();
* O3List.AddRange(obsDescriptionList.Select(observationDescription => observationDescription.O3));
*
* //ALL descriptors mapped from the SDTM entity descriptor
* var descriptorsList = LoadObsDescriptors(sdtmEntityDescriptor, dataset.Activity.ProjectId);
*
* foreach (var sdtmRow in sdtmRowList)
* {
* //WARNING
* //depending ON THE STRING EQUALITY HERE TO IDENTIFY O3 ACROSS DATASETS AND ACROSS STUDIES OF THE SAME PROJECT
* //SHOULD REALLY USE THE CONTROLLED TERM INSTEAD (I.E. DECODE / LOINC)
*
* //1- GET O3
* ObjectOfObservation O3obj;
* string O3 = sdtmRow.Topic; //THIS IS --TESTCD AND --TERM FOR NOW
* if (sdtmRow.Group != null && sdtmRow.Subgroup != null)
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group)
* && o3.Subgroup.Equals(sdtmRow.Subgroup));
* else if (sdtmRow.Group != null)
* {
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode)
* && o3.Group.Equals(sdtmRow.Group));
* }
* else
* O3obj = O3List.Find(o3 => o3.Name.Equals(O3)
* && o3.ProjectId.Equals(dataset.Activity.ProjectId)
* && o3.Domain.Code.Equals(sdtmRow.DomainCode));
*
*
* //2- Get its observationDefinition
* var obsDescription = obsDescriptionList.Find(od => od.ObjectOfObservationId.Equals(O3obj.Id));
*
* //3- Create new observation
* Study study = _studyRepository.FindSingle(s => s.Name.Equals(sdtmRow.StudyId));
* var observation = new Core.Domain.Model.ObservationModel.Observation();
* observation.O3Id = O3obj.Id;
* observation.ObjectOfObservation = O3obj.Name;
* observation.Study = sdtmRow.StudyId;
* observation.StudyDBId = study != null ? study.Id : 0;
* observation.SO2Id = sdtmRow.USubjId;
* observation.SO2Type = "SUBJECT";
*
*
#region 3.1 Add Default property value
*
* //THE DEFAULT DESCRIPTOR IS NOT A NATIVE DEFAULT DESCRIPTOR PRE-SPECIFIED ,
* //IT'S just a regular descriptor which could take on any qualifier value
* //e.g. OCCUR, ORRES, STRESC, STRESN
* // and only at the time of creating the obsevation first time the observationDefinition.DefaultQualifier will be set to one of the qualifiers
* //based on ... Cannot be based on information pre-assigned to the SDTMentity descriptor since that applies to the dataset as a whole and not on a row
* //basis (will not solve the situation where some observations will have their defualt set to STRESN and others set to STRESC)
* // so at the time of observation creation first time based on the following rules set its observationDefinition.DefaultQualifier
* // if domain is findings and if there's a value for STRESN then this is the default,
* //else if there's a value for STRESC then this is default else ORRES is the default
* //this equality is going to be achieved
*
*
* VariableDefinition defQVariable = null; //the variable that will be read from the sdtm row as the default
* PropertyDescriptor defDescriptor; //= obsDescription.DefaultPropertyDescriptor; //the matching descriptor and set it to observationDescription
*
* if (dataset.Domain.Class.ToLower().Equals("findings"))
* {
* var defVar = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESN"));
* if (sdtmRow.ResultQualifiers["STRESN"] != null)
* {
* defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESN"));
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("STRES"));
* }
* else if (sdtmRow.ResultQualifiers["STRESC"] != null)
* {
* defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESC"));
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("STRES"));
* }
* else //if (sdtmRow.ResultQualifiers["ORRES"] != null)
* {
* defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("ORRES"));
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("ORRES"));
* }
* }
* else if (sdtmEntityDescriptor.DomainCode.Equals("AE"))
* {
* defDescriptor = descriptorsList.Find(d => d.Name.Equals("OCCUR"));
* //there is no defQvariable in this case as we will make it up it doesn not occur in the original dataset
* }
* else
* {
* defQVariable = sdtmEntityDescriptor.DefaultQualifier;
* defDescriptor = descriptorsList.Find(d => d.Name.Equals(defQVariable.Name));
* }
*
* obsDescription.DefaultPropertyDescriptor = defDescriptor;
*
* //TODO: THIS COULD EITHER BE CVTERMS SO BOTH VARIABLE DEFINITION AND OBSERVATION DESCRIPTOR WOULD SHARE THE SAME CVTERM
* //todo: CHANGE ALL VARIABLE NAMES TO ENUMS
*
*
* if (sdtmEntityDescriptor.DomainCode.Equals("AE"))
* {
* CategoricalValue propValue = new CategoricalValue();
* propValue.Property = defDescriptor;
* propValue.Value = "Y";
* }
* else if (defDescriptor.ValueType == ObsValueType.Numerical)
* {
* NumericalValue propertyValue = new NumericalValue();
* propertyValue.Value = float.Parse(sdtmRow.ResultQualifiers[defQVariable.Name]);
* //propertyValue.Unit =
* propertyValue.Property = defDescriptor;
* observation.DefaultObservedProperty = propertyValue;
* }
* else
* {
* CategoricalValue propValue = new CategoricalValue();
* propValue.Property = defDescriptor;
* propValue.Value = sdtmRow.ResultQualifiers[defQVariable.Name];
* }
#endregion
*
* ///////////////////////////////////////////////////////////////////////////////////////////
*
#region 3.2 Add other qualifiers
*
* foreach (var qualifier in sdtmRow.Qualifiers)
* {
* var key = qualifier.Key.Substring(2);
* //GET THE MATCHING VARIABLE FROM the sdtm desriptor using names qualifier.key ==
* //then use its name
* if (qualifier.Value != null)
* {
* //ADD descriptor to obsDescription if not previously added
* //HOW IS THE LINK BETWEEN THE VALUE IN THE SDTM ENTITY AND THE DESCRIPTOR IS ESTABLISHED
* //CURRENTLY IT IS USING THE SAME NAME FOR THE DESCRIPTOR AS THE NAME OF THE VARIABLE WITHOUT THE PREFIX
*
* PropertyDescriptor descriptor = obsDescription.ObservedPropertyDescriptors.Find(d => d.Name.Equals(key));
* if (descriptor == null)
* {
* descriptor = descriptorsList.Find(d => d.Name.Equals(key));
* obsDescription.ObservedPropertyDescriptors.Add(descriptor);
* }
* //ADD new observation instance
*
* if(descriptor.ValueType.Equals(ObsValueType.Categorical))
* //TODO: for now not checking on property value type and creating categorical value for all other than result for findings
* observation.ObservedProperties.Add(new CategoricalValue()
* {
* Value = qualifier.Value,
* Property = descriptor,
*
* //CVTerm =
* });
*
*
*
* }
* }
*
#endregion
*
* ///////////////////////////////////////////////////////////////////////////////////////////
*
#region 4 Add Temporal Qualifiers
*
#endregion
*
#region 5 Add TimeSeries Descriptors
*
#endregion
* }
* }
*
* internal List<PropertyDescriptor> LoadObsDescriptors(SdtmRowDescriptor sdtmEntityDescriptor, int projectId)
* {
*
* var descriptorList = new List<PropertyDescriptor>();
*
* // SetDescriptorTypes(sdtmEntityDescriptor);
*
* //IF DOMAIN.CLASS which can be identified from sdtmEntityDescriptor
* //THEN cREATE descriptor for MATCHIN CLASS
* //BUT then how will this be linked with the variable name in the data file?
*
#region HACK
* //TODO: chage this to enum
*
* //A HACK TO ADD OCCURENCE descriptor for adverse events which would not be in the standard
* if (sdtmEntityDescriptor.DomainCode.Equals("AE"))
* {
* var descriptor = _ObsDescriptorRepository.FindSingle(d => d.Name.Equals("OCCUR"));
* //o => o.Type == DescriptorType.
* //&& o.Domain.Equals(sdtmEntityDescriptor.DomainCode));
*
* if (descriptor == null)
* {
* descriptor = PropertyDescriptorFactory.CreateAEdefDescriptor();
* _ObsDescriptorRepository.Insert(descriptor);
* }
* descriptorList.Add(descriptor);
* }
*
* //A HACK BECAUSE I DONT WANT TO HAVE TWO STANDARD RESULT DESCRIPTORS AS IT IS THE CASE WITH SDTM VARAIBLES
* if (sdtmEntityDescriptor.Class.Equals("Findings"))
* {
* var descriptorST = _ObsDescriptorRepository.FindSingle(d => d.Name.Equals("STRES"));
* if (descriptorST == null)
* {
* descriptorST = PropertyDescriptorFactory.CreateStandardResDescriptor();
* _ObsDescriptorRepository.Insert(descriptorST);
* }
* descriptorList.Add(descriptorST);
* }
#endregion
*
* foreach (var variable in sdtmEntityDescriptor.QualifierVariables.Union(sdtmEntityDescriptor.ResultVariables))
* {
* //todo: SHOULD BE ABLE TO DIFFERENTIATE HERE THE DIFFERENT SUBCLASSES OF SDTM RECORD QUALIFIERS
* //TODO: INTO FEATURE_QUALIFIERS, DEFAULT_OBSERVED_PROPERTY, OBSERVED_PROPERTIES, SUBJECT_QUALIFIERS AND SAMPLE_QUALIFIERS
* //TODO: should find equality via CVterm not name text
* //TODO: CVterm should be assigned to each variable definition which shuold then be an EQUIVALENT CVterm used for the descriptor
* var descriptor = _ObsDescriptorRepository.FindSingle(o => o.Name.Equals(variable.Name.Substring(2)));
* if (descriptor == null)
* {
* //FOR NOW the Descriptor is CREATED FROM the variable information
* descriptor = new PropertyDescriptor();
* descriptor.Name = variable.Name.Substring(2);
* //SOME KIND OF A METHOD HERE THAT RETURNS THE DESCRIPTOR TYPE THIS VARIABLE MATCHES
* //descriptor.Type = variable.DescriptorType; //THIS IS SUPPOSED TO HAVE NOW THE NEW SUBCLASSES //DescriptorType.ObservedPropertyDescriptor;
* descriptor.Description = variable.Label;
* descriptor.ProjectId = projectId;
*
* //_ObsDescriptorRepository.Insert(descriptor);
* }
* descriptorList.Add(descriptor);
* }
*
* //_dataContext.Save();
*
* //TODO: consider doing the same for other qualifiers such as timings and setting variable qualifiers to qualifiers
*
* return descriptorList;
* }
*/
internal bool LoadObservations(List <SdtmRow> sdtmData, SdtmRowDescriptor sdtmRowDescriptor, bool reload)
{
var dsDomainCode = sdtmRowDescriptor.DomainCode;
var dsClass = sdtmRowDescriptor.Class;
var datasetId = sdtmData.First().DatasetId;
var dataFileId = sdtmData.First().DatafileId;
var projectId = sdtmData.First().ProjectId;
if (reload)
{
_ObservationRepository.DeleteMany(o => o.DatasetId == datasetId && o.DatafileId == dataFileId);
_dataContext.Save().Equals("CREATED");
}
//Retrieve ObjectOfObservations previously loaded for this project
// CURRENTLY THIS IS NOT REALLY THE OBJECTOFOBSERVATION BUT THE OBSERVATION DESCRIPTOR WHICH IS
//DEFINED PER DATASET/DATAFILE AND IS DELETABLE ACROSS LOADS AS LONG AS THE O3 CV IS UNIQUE ACROSS DATASETS
//AND NOT JUST INFLUENCED BY ONE DATASET OR THE FIRST DATASET LOADED AS IT IS NOW!!
var projectO3s = _ObservationRepository.FindAll(o => o.ProjectId == projectId && o.DomainCode == dsDomainCode).ToList();
List <string> O3keys = projectO3s.Select(currObservation => currObservation.Class + currObservation.DomainCode + currObservation.Group + currObservation.Name).ToList();
var observations = sdtmData.GroupBy(o => new { domain = o.DomainCode, o3 = o.Topic, group = o.Group, subgroup = o.Subgroup, o3CVterm = o.TopicControlledTerm ?? o.TopicSynonym });
foreach (var observation in observations)
{
var O3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3;
if (O3keys.Contains(O3key))
{
continue;
}
Observation obsDescriptor = new Observation();
obsDescriptor.Name = observation.Key.o3;
obsDescriptor.Group = observation.Key.group;
obsDescriptor.Subgroup = observation.Key.subgroup;
obsDescriptor.Class = sdtmRowDescriptor.Class;
obsDescriptor.DomainCode = sdtmRowDescriptor.DomainCode;
obsDescriptor.DomainName = sdtmRowDescriptor.Domain;
obsDescriptor.TopicVariable = sdtmRowDescriptor.TopicVariable;
if (sdtmRowDescriptor.ObsIsAFinding)
{
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.ResultVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor, Qualifier = qualifier
}));
}
//sdtmRowDescriptor.ResultVariables.Union(sdtmRowDescriptor.QualifierVariables).ToList())};
else
{
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.QualifierVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
}
obsDescriptor.Timings.AddRange(sdtmRowDescriptor.GetAllTimingVariables().FindAll(v => v != null).Select(
qualifier => new ObservationTiming()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
obsDescriptor.ControlledTermStr = observation.Key.o3CVterm;
obsDescriptor.DatasetId = datasetId;
obsDescriptor.DatafileId = dataFileId;
obsDescriptor.ProjectId = projectId;
obsDescriptor.DefaultQualifier = sdtmRowDescriptor.GetDefaultQualifier(observation.FirstOrDefault());
// //LOAD CVTERMS CORRESPONDING TO MEDdRA codes
// ob = CreateObsCVterms(observation.FirstOrDefault(), sdtmRowDescriptor, out ob);
// ob.ControlledTermId = observation.FirstOrDefault().QualifierQualifiers?[AECVtermIdVar];
//}
_ObservationRepository.Insert(obsDescriptor);
}
var success = _dataContext.Save().Equals("CREATED");
//addO3IdstoSDTMrows()
if (success)
{
//TODO: PROBLEM!!
//IN CASE OF A SECOND FILE LOADED TO A PREVIOUSLY CREATED DATASET, DATAFILE WILL BE DIFFERENT
//OBSERVATIONS WERE SAVED WITH THE FISRT DATASETID AND THE FIRST DATAFILE
//WHAT HAPPENS WHERE WE WANT TO UNLOAD A FILE THAT BROUGHT DIFFERENT O3s to the DATASET DIFFERENT
//FROM THE FIRST DATAFILE?
//I STILL NEED TO BE ABLE TO FIND O3s that were loaded from a certain DATAFILE
var savedObs = _ObservationRepository.FindAll(o => o.DatasetId == datasetId && o.DatafileId == dataFileId).ToList();
foreach (var observation in observations)
{
foreach (var sdtmRow in observation)
{
var O3 =
savedObs.FirstOrDefault(
o => o.Name == sdtmRow.Topic && o.Group == sdtmRow.Group && o.Subgroup == sdtmRow.Subgroup);
if (O3 != null)
{
sdtmRow.DBTopicId = O3.Id;
}
_sdtmRepository.Update(sdtmRow);
}
}
}
return(success);
}
public async Task <bool> LoadObservations(Dataset dataset, int fileId, bool reload)
{
var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(dataset);
List <SdtmRow> sdtmData = await _sdtmRepository.FindAllAsync(
dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId));
var dsDomainCode = sdtmRowDescriptor.DomainCode;
var dsClass = sdtmRowDescriptor.Class;
var datasetId = sdtmData.First().DatasetId;
var dataFileId = sdtmData.First().DatafileId;
var projectId = sdtmData.First().ProjectId;
if (reload)
{
_observationRepository.DeleteMany(o => o.DatasetId == datasetId && o.DatafileId == dataFileId);
_dataContext.Save();
}
//Retrieve ObjectOfObservations previously loaded for this project
// CURRENTLY THIS IS NOT REALLY THE OBJECTOFOBSERVATION BUT THE OBSERVATION DESCRIPTOR WHICH IS
//DEFINED PER DATASET/DATAFILE AND IS DELETABLE ACROSS LOADS AS LONG AS THE O3 CV IS UNIQUE ACROSS DATASETS
//AND NOT JUST INFLUENCED BY ONE DATASET OR THE FIRST DATASET LOADED AS IT IS NOW!!
var projectO3s = _observationRepository.FindAll(o => o.ProjectId == projectId && o.DomainCode == dsDomainCode).ToList();
//List<string> O3keys =
// projectO3s.Select(
// currObservation =>
// currObservation.Class + currObservation.DomainCode + currObservation.Group +
// currObservation.Name).ToList();
var O3map = projectO3s.ToDictionary(o3 => o3.Class + o3.DomainCode + o3.Group + o3.Name + o3.ControlledTermStr);
var observations =
sdtmData.GroupBy(
o =>
new
{
domain = o.DomainCode,
o3 = o.Topic,
group = o.Group,
subgroup = o.Subgroup,
o3CVterm = o.TopicControlledTerm ?? o.TopicSynonym
});
var obsPrevLoaded = new Dictionary <string, Observation>();
foreach (var observation in observations)
{
var O3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3 + observation.Key.o3CVterm;
Observation o3;
if (O3map.TryGetValue(O3key, out o3))
{
obsPrevLoaded.Add(O3key, o3);
continue;
}
Observation obsDescriptor = new Observation();
obsDescriptor.Name = observation.Key.o3;
obsDescriptor.Group = observation.Key.group;
obsDescriptor.Subgroup = observation.Key.subgroup;
obsDescriptor.Class = sdtmRowDescriptor.Class;
obsDescriptor.DomainCode = sdtmRowDescriptor.DomainCode;
obsDescriptor.DomainName = sdtmRowDescriptor.Domain;
obsDescriptor.TopicVariable = sdtmRowDescriptor.TopicVariable;
if (sdtmRowDescriptor.ObsIsAFinding)
{
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.ResultVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
}
//sdtmRowDescriptor.ResultVariables.Union(sdtmRowDescriptor.QualifierVariables).ToList())};
obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.QualifierVariables.Select(
qualifier => new ObservationQualifier()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
obsDescriptor.Timings.AddRange(sdtmRowDescriptor.TimeVariables.Select(
qualifier => new ObservationTiming()
{
Observation = obsDescriptor,
Qualifier = qualifier
}));
obsDescriptor.ControlledTermStr = observation.Key.o3CVterm;
obsDescriptor.DatasetId = datasetId;
obsDescriptor.DatafileId = dataFileId;
obsDescriptor.ProjectId = projectId;
obsDescriptor.DefaultQualifier = sdtmRowDescriptor.GetDefaultQualifier(observation.FirstOrDefault());
var newObs = _observationRepository.Insert(obsDescriptor);
obsPrevLoaded.Add(O3key, newObs);
}
var success = _dataContext.Save().Equals("CREATED");
if (success)
{
//TODO: PROBLEM!!
//IN CASE OF A SECOND FILE LOADED TO A PREVIOUSLY CREATED DATASET, DATAFILE WILL BE DIFFERENT
//OBSERVATIONS WERE SAVED WITH THE FISRT DATASETID AND THE FIRST DATAFILE
//WHAT HAPPENS WHERE WE WANT TO UNLOAD A FILE THAT BROUGHT DIFFERENT O3s to the DATASET DIFFERENT
//FROM THE FIRST DATAFILE?
//I STILL NEED TO BE ABLE TO FIND O3s that were loaded from a certain DATAFILE
var savedObs = _observationRepository.FindAll(o => o.DatasetId == datasetId && o.DatafileId == dataFileId).ToList();
foreach (var observation in observations)
{
foreach (var sdtmRow in observation)
{
Observation O3;
var o3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3 + observation.Key.o3CVterm;
if (obsPrevLoaded.TryGetValue(o3key, out O3))
{
sdtmRow.DBTopicId = O3.Id;
}
_sdtmRepository.Update(sdtmRow);
}
}
}
return(success);
}
public bool LoadSDTM(int datasetId, int fileId, DataTable dataTable)
{
var dataset = _datasetRepository.FindSingle(d => d.Id == datasetId, new List <string>()
{
"Variables.VariableDefinition", "Template", "Activity.Project"
});
var dataFile = _dataFileRepository.Get(fileId);
if (!dataFile.State.ToLower().Equals("new"))
{
/**
* Replacing previously loaded file
* Remove file from collection before reloading it
*/
_sdtmRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId);
Debug.WriteLine("RECORD(s) SUCCESSFULLY DELETED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId);
}
var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(dataset);
var SDTM = new List <SdtmRow>();
var totalLoaded = 0.0;
try
{
foreach (DataRow row in dataTable.Rows)
{
var sdtmRow = SDTMreader.readSDTMrow(row, dataTable, sdtmRowDescriptor);
sdtmRow.Id = Guid.NewGuid();
sdtmRow.DatasetId = datasetId;
sdtmRow.ActivityId = dataset.ActivityId;
sdtmRow.DatafileId = fileId;
sdtmRow.ProjectId = dataset.Activity.ProjectId;
sdtmRow.ProjectAccession = dataset.Activity.Project.Accession;
SDTM.Add(sdtmRow);
if (SDTM.Count % 100 == 0)
{
_sdtmRepository.InsertMany(SDTM);
totalLoaded += SDTM.Count;
dataFile.State = Math.Round(totalLoaded / dataTable.Rows.Count * 100).ToString("##");
_dataFileRepository.Update(dataFile);
_dataServiceUnit.Save();
SDTM.Clear();
}
}
_sdtmRepository.InsertMany(SDTM);
Debug.WriteLine(dataTable.Rows.Count + " RECORD(s) SUCCESSFULLY ADDED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId);
}
catch (Exception e)
{
dataFile.State = "FAILED";
dataFile.IsLoadedToDB = false;
_dataFileRepository.Update(dataFile);
_dataServiceUnit.Save();
Debug.WriteLine(e.Message);
return(false);
}
dataFile.State = "SAVED";
_dataFileRepository.Update(dataFile);
_dataServiceUnit.Save();
return(true);
}
