public DataFile CreateFolder(int projectId, DirectoryDTO folder) { //if (Directory.Exists(newDir)) //return new DirectoryInfo(newDir); //var di = Directory.CreateDirectory(newDir); var project = _projectRepository.FindSingle(p => p.Id == projectId); if (project == null || folder.Name == "" || folder.Name == null) { return(null); } var file = new DataFile(); file.FileName = folder.Name; file.DateAdded = DateTime.Now.ToString("D"); file.IsDirectory = true; file.ProjectId = project.Id; if (folder.ParentFolderId != 0) { file.FolderId = folder.ParentFolderId; } _fileRepository.Insert(file); return(_dataServiceUnit.Save().Equals("CREATED") ? file : null); }
public DirectoryInfo AddDirectory(int projectId, string newDir) { if (Directory.Exists(newDir)) { return(new DirectoryInfo(newDir)); } var di = Directory.CreateDirectory(newDir); var project = _projectRepository.FindSingle(p => p.Id == projectId); if (project == null) { return(null); } var file = new DataFile(); file.FileName = di.Name; //file.Path = di.FullName.Substring(di.FullName.IndexOf(projectId)); file.Path = di.Parent.FullName.Substring(di.Parent.FullName.IndexOf("P-" + projectId));//file.Path.Substring(0,file.Path.LastIndexOf("\\\")) file.DateAdded = di.CreationTime.ToString("D"); file.IsDirectory = true; file.ProjectId = project.Id; _fileRepository.Insert(file); return(_dataServiceUnit.Save().Equals("CREATED") ? di : null); }
public Dataset addDataset(DatasetDTO datasetDTO) { //1. Fields for Dataset var dataset = CreateDataset(datasetDTO); _datasetRepository.Insert(dataset); return(_dataServiceUnit.Save().Equals("CREATED") ? dataset : null); }
public void DeleteProject(int projectId) { var projectToDelete = _projectRepository.Get(projectId); _projectRepository.Remove(projectToDelete); _sdtmRepository.DeleteMany(s => s.ProjectId == projectId); uoW.Save(); }
public AssayDTO AddAssay(AssayDTO assayDto) { var assay = new Assay(); //var project = _projectRepository.FindSingle(d => d.Accession // .Equals(assayDto.ProjectAcc)); assay.Name = assayDto.Name; assay.ProjectId = assayDto.ProjectId; assay.TechnologyPlatformId = assayDto.Platform; assay.TechnologyTypeId = assayDto.Technology; //assay.DesignType = getCVterm(assayDto.AssayDesignType); assay.MeasurementTypeId = assayDto.Type; if (assayDto.SamplesDataset != null) { assayDto.SamplesDataset.ProjectId = assayDto.ProjectId; } if (assayDto.FeaturesDataset != null) { assayDto.FeaturesDataset.ProjectId = assayDto.ProjectId; } if (assayDto.ObservationsDataset != null) { assayDto.ObservationsDataset.ProjectId = assayDto.ProjectId; } var BSdataset = _datasetService.CreateDataset(assayDto.SamplesDataset); if (BSdataset != null) { assay.Datasets.Add(BSdataset); } var FEdataset = _datasetService.CreateDataset(assayDto.FeaturesDataset); if (FEdataset != null) { assay.Datasets.Add(FEdataset); } var OBdataset = _datasetService.CreateDataset(assayDto.ObservationsDataset); if (OBdataset != null) { assay.Datasets.Add(OBdataset); } assay = _assayRepository.Insert(assay); if (_dataContext.Save().Equals("CREATED")) { assayDto.Id = assay.Id; return(assayDto); } return(null); }
//public void addDatasetVariables(List<VariableDefinition> variableDefinitions) //{ // for (int i = 0; i < variableDefinitions.Count; i++) // { // _variableDefinitionRepository.Insert(variableDefinitions[i]); // } // _studyServiceUnit.Save(); //} public string Updatestudy(StudyDTO studyDto, int studyId) { var studyToUpdate = _studyRepository.Get(studyId); studyToUpdate.Name = studyDto.Name; studyToUpdate.Description = studyDto.Title; _studyRepository.Update(studyToUpdate); return(_studyServiceUnit.Save()); }
public void LoadDictionaries() { var reader = File.OpenText(Path.Combine(_JSONDir, _termsJSONfilename)); List <Dictionary> dicts = JsonConvert.DeserializeObject <List <Dictionary> >(reader.ReadToEnd()); _dictRepository.InsertMany(dicts); _db.Save(); reader.Dispose(); }
public ActivityDTO AddActivity(ActivityDTO activityDTO) { //var project = _projectRepository.FindSingle(d => d.Accession // .Equals(activityDTO.ProjectAcc)); var activity = new Activity { Name = activityDTO.Name, ProjectId = activityDTO.ProjectId }; foreach (var datasetDto in activityDTO.datasets) { datasetDto.ProjectId = activityDTO.ProjectId; var dataset = _datasetService.CreateDataset(datasetDto); activity.Datasets.Add(dataset); } activity = _activityRepository.Insert(activity); if (_activityServiceUnit.Save().Equals("CREATED")) { activityDTO.Id = activity.Id; return(activityDTO); } return(null); }
public UserDatasetDTO AddUserDataset(UserDatasetDTO dto, string userId) { var NewDS = ReadDTO(dto); NewDS.OwnerId = userId; //var project = _projectRepository.FindSingle(p => p.Accession == dto.ProjectAcc); NewDS.ProjectId = dto.ProjectId; NewDS.Id = Guid.NewGuid(); var addedUserDataset = _userDatasetRepository.Insert(NewDS); if (!_unitOfWork.Save().Equals("CREATED")) { return(null); } dto.Id = addedUserDataset.Id.ToString(); return(dto); }
internal bool LoadHDdata(int datasetId, int fileId, DataTable dataTable) { var dataset = _datasetRepository.FindSingle(d => d.Id == datasetId); var dataFile = _dataFileRepository.Get(fileId); //if (!dataFile.State.ToLower().Equals("new")) //{ // /** // * Replacing previously loaded file // * Remove file from collection before reloading it // */ // _observationRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId); // Debug.WriteLine("RECORD(s) SUCCESSFULLY DELETED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId); //} var projectId = dataFile.ProjectId; var observations = new List <Observation>(); var totalLoaded = 0.0; var prop = new PropertyDescriptor() { Name = "CONC", ProjectId = projectId, Description = "Concentration", ObsClass = "ASSAYMEASURES", Id = 99 }; try { foreach (DataRow row in dataTable.Rows) { var obs = new Observation(); obs.FeatureName = row["FEAT"]?.ToString(); // obs.SubjectOfObservationId = row["SPECIMEN"]?.ToString(); obs.SubjectOfObservationName = row["SPECIMEN"]?.ToString(); double val; if (double.TryParse(row["OBSVAL"]?.ToString(), out val)) { obs.ObservedValue = new NumericalValue() { Property = prop, Unit = row["OBSVALU"]?.ToString(), Value = val } } ; else { obs.ObservedValue = new MissingValue() { Property = prop }; } obs.Id = Guid.NewGuid(); obs.DatasetId = datasetId; obs.ActivityId = dataset.ActivityId; obs.DatafileId = fileId; obs.ProjectId = projectId; observations.Add(obs); if (observations.Count % 500 == 0) { _observationRepository.InsertMany(observations); totalLoaded += observations.Count; dataFile.State = Math.Round(totalLoaded / dataTable.Rows.Count * 100).ToString("##"); _dataFileRepository.Update(dataFile); _dataContext.Save(); observations.Clear(); } } _observationRepository.InsertMany(observations); Debug.WriteLine(dataTable.Rows.Count + " RECORD(s) SUCCESSFULLY ADDED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId); } catch (Exception) { dataFile.State = "FAILED"; dataFile.IsLoadedToDB = false; _dataFileRepository.Update(dataFile); _dataContext.Save(); return(false); } dataFile.State = "LOADED"; dataFile.IsLoadedToDB = true; _dataFileRepository.Update(dataFile); _dataContext.Save(); return(true); }
// dataset service public bool LoadHDdDdata(int datasetId, int fileId /*, int referencFromHdId*/) { _observationRepository.DeleteMany(d => d.DatasetId == datasetId); var dataset = GetActivityDataset(datasetId); var dataFile = _dataFileRepository.Get(fileId); var filePath = dataFile.Path + "\\" + dataFile.FileName; var dataTable = _fileService.ReadOriginalFile(filePath); var sampleList = _biosampleRepository.FindAll(s => s.DatasetId == datasetId).ToList(); var scos = sampleList.ToDictionary(co => co.BiosampleStudyId); var obsReadyToInsert = new List <Observation>(); foreach (DataRow row in dataTable.Rows) { for (int index = 0; index < dataTable.Columns.Count; index++) { if (index == 0) { continue; } else { var column = dataTable.Columns[index]; var obs = new Observation(); // Fill an Observation var value = new NumericalValue(); value.Value = double.Parse(row[column.ColumnName].ToString()); value.Property = new PropertyDescriptor(); { value.Property.Description = dataset.Template.Description; value.Property.ObsClass = dataset.Template.Class; value.Property.Name = dataset.TemplateId; } // value.Unit = ?? obs.ObservedValue = value; // Observed Value which includes in it the VALUE and its PROPERTY // ****** FOR NOW WE USE THE FOLLOWING TILL WE HAVE THE FEATURE referencFromHdId ****** obs.SubjectOfObservationName = column.ColumnName; // sample name (HERE IS THE FILE NAME) //obs.SubjectOfObservationId = column.ColumnName; // sampleID (here is the file name) // ****** // obs.SubjectOfObservationName = scos[column.ColumnName].BiosampleStudyId; // sample Name // obs.SubjectOfObservationId = scos[column.ColumnName].Id.ToString(); // sample ID // obs.StudyId = scos[column.ColumnName].StudyId; // study ID obs.FeatureName = row[0].ToString(); // feature name // obs.FeatureId =; Not implemented yet // feature ID obs.DatafileId = dataFile.Id; obs.DatasetId = datasetId; obs.ActivityId = dataset.ActivityId; obs.ProjectId = dataset.Activity.ProjectId; obs.Id = Guid.NewGuid(); obsReadyToInsert.Add(obs); // observationsTotal.Add(obs); if (obsReadyToInsert.Count % ((dataTable.Columns.Count) - 1) == 0) { _observationRepository.InsertMany(obsReadyToInsert); obsReadyToInsert.Clear(); } } } } if (obsReadyToInsert.Count > 0) { Debug.WriteLine("Created Observations are NOT Equal to measured values in the file please check!!"); } dataFile.State = "LOADED"; dataFile.IsLoadedToDB = true; _dataFileRepository.Update(dataFile); _dataContext.Save(); return(true); }
/* * public void LoadObservations(Dataset dataset, SdtmRowDescriptor sdtmEntityDescriptor, List<SdtmRow> sdtmRowList) * { * * //Retrieve all Observtion Descriptors for this domain and this project if previously a dataset of the same domain has been loaded? * var obsDescriptionList = _ObsDescriptionRepository.FindAll( * s => s.ProjectId.Equals(dataset.Activity.ProjectId) && s.DomainId.Equals(dataset.DomainId), * new List<Expression<Func<ObservationDescriptor, object>>>() { od => od.O3 }).ToList(); * * var O3List = new List<ObjectOfObservation>(); * if (obsDescriptionList.Count != 0) * O3List.AddRange(obsDescriptionList.Select(observationDescription => observationDescription.O3)); * #region Create O3 and ObservationDescription * foreach (var sdtmRow in sdtmRowList) * { * //O3 name is not enough to uniquely identify the observation in place * //e.g. white blood cells with two categories Heamatology and Chemistry * //TODO: if an observation was previoulsy loaded via a dataset that had no group for it * //e.g. white blood cells that was meant to be heamtology but not explicitly mentioned * //then the same white blood cells is found but with a group, it will be loaded as a different O3 * //In any case, the CVterm of whitebloodcells for heamatology should be different from whitebloodcell chemsitry * //ACTUALLY TOPIC variable SHOULD not be white blood cells but rather the TEST itself which in this case would be * //Heamtatology test * string O3 = sdtmRow.Topic; //THIS IS --TESTCD AND --TERM FOR NOW * ObjectOfObservation O3obj; * ObservationDescriptor obsDescriptor; * * * * if (sdtmRow.Group != null && sdtmRow.Subgroup != null) * O3obj = O3List.Find(o3 => o3.Name.Equals(O3) * && o3.ProjectId.Equals(dataset.Activity.ProjectId) * && o3.Domain.Code.Equals(sdtmRow.DomainCode) * && o3.Group.Equals(sdtmRow.Group) * && o3.Subgroup.Equals(sdtmRow.Subgroup)); * else if (sdtmRow.Group != null) * { * O3obj = O3List.Find(o3 => o3.Name.Equals(O3) * && o3.ProjectId.Equals(dataset.Activity.ProjectId) * && o3.Domain.Code.Equals(sdtmRow.DomainCode) * && o3.Group.Equals(sdtmRow.Group)); * } * else * O3obj = O3List.Find(o3 => o3.Name.Equals(O3) * && o3.ProjectId.Equals(dataset.Activity.ProjectId) * && o3.Domain.Code.Equals(sdtmRow.DomainCode)); * if (O3obj != null) continue; * * //Create new O3 * O3obj = new ObjectOfObservation(); * O3obj.Name = sdtmRow.Topic; //EOS * O3obj.Synonym = sdtmRow.TopicSynonym ?? null; //Eosinophils * O3obj.ProjectId = dataset.Activity.ProjectId; //id for P-BVS * O3obj.DomainId = dataset.DomainId; //id for D-SDTM-LB * O3obj.CVTermStr = sdtmRow.TopicControlledTerm ?? null; //TEMP //Value of DECOD or LOINC * //O3obj.CVtermId = lookupCVterm(sdtmRow.TopicVariableControlledTerm); * O3obj.Group = sdtmRow.Group; //Hematology * O3obj.Subgroup = sdtmRow.Subgroup; //value of SCAT * _O3Repository.Insert(O3obj); * O3List.Add(O3obj); * * //Create ObservationDescriptor * obsDescriptor = new ObservationDescriptor(); * obsDescriptor.O3 = O3obj; * obsDescriptor.DomainId = dataset.DomainId; * obsDescriptor.ProjectId = dataset.Activity.ProjectId; * * _ObsDescriptionRepository.Insert(obsDescriptor); * * } * _dataContext.Save(); #endregion * * obsDescriptionList = _ObsDescriptionRepository.FindAll( * s => s.ProjectId.Equals(dataset.Activity.ProjectId) && s.DomainId.Equals(dataset.DomainId), * new List<Expression<Func<ObservationDescriptor, object>>>() { od => od.O3 }).ToList(); * * //NOW WITH IDS TO PASS ON TO MONGO * O3List = new List<ObjectOfObservation>(); * O3List.AddRange(obsDescriptionList.Select(observationDescription => observationDescription.O3)); * * //ALL descriptors mapped from the SDTM entity descriptor * var descriptorsList = LoadObsDescriptors(sdtmEntityDescriptor, dataset.Activity.ProjectId); * * foreach (var sdtmRow in sdtmRowList) * { * //WARNING * //depending ON THE STRING EQUALITY HERE TO IDENTIFY O3 ACROSS DATASETS AND ACROSS STUDIES OF THE SAME PROJECT * //SHOULD REALLY USE THE CONTROLLED TERM INSTEAD (I.E. DECODE / LOINC) * * //1- GET O3 * ObjectOfObservation O3obj; * string O3 = sdtmRow.Topic; //THIS IS --TESTCD AND --TERM FOR NOW * if (sdtmRow.Group != null && sdtmRow.Subgroup != null) * O3obj = O3List.Find(o3 => o3.Name.Equals(O3) * && o3.ProjectId.Equals(dataset.Activity.ProjectId) * && o3.Domain.Code.Equals(sdtmRow.DomainCode) * && o3.Group.Equals(sdtmRow.Group) * && o3.Subgroup.Equals(sdtmRow.Subgroup)); * else if (sdtmRow.Group != null) * { * O3obj = O3List.Find(o3 => o3.Name.Equals(O3) * && o3.ProjectId.Equals(dataset.Activity.ProjectId) * && o3.Domain.Code.Equals(sdtmRow.DomainCode) * && o3.Group.Equals(sdtmRow.Group)); * } * else * O3obj = O3List.Find(o3 => o3.Name.Equals(O3) * && o3.ProjectId.Equals(dataset.Activity.ProjectId) * && o3.Domain.Code.Equals(sdtmRow.DomainCode)); * * * //2- Get its observationDefinition * var obsDescription = obsDescriptionList.Find(od => od.ObjectOfObservationId.Equals(O3obj.Id)); * * //3- Create new observation * Study study = _studyRepository.FindSingle(s => s.Name.Equals(sdtmRow.StudyId)); * var observation = new Core.Domain.Model.ObservationModel.Observation(); * observation.O3Id = O3obj.Id; * observation.ObjectOfObservation = O3obj.Name; * observation.Study = sdtmRow.StudyId; * observation.StudyDBId = study != null ? study.Id : 0; * observation.SO2Id = sdtmRow.USubjId; * observation.SO2Type = "SUBJECT"; * * #region 3.1 Add Default property value * * //THE DEFAULT DESCRIPTOR IS NOT A NATIVE DEFAULT DESCRIPTOR PRE-SPECIFIED , * //IT'S just a regular descriptor which could take on any qualifier value * //e.g. OCCUR, ORRES, STRESC, STRESN * // and only at the time of creating the obsevation first time the observationDefinition.DefaultQualifier will be set to one of the qualifiers * //based on ... Cannot be based on information pre-assigned to the SDTMentity descriptor since that applies to the dataset as a whole and not on a row * //basis (will not solve the situation where some observations will have their defualt set to STRESN and others set to STRESC) * // so at the time of observation creation first time based on the following rules set its observationDefinition.DefaultQualifier * // if domain is findings and if there's a value for STRESN then this is the default, * //else if there's a value for STRESC then this is default else ORRES is the default * //this equality is going to be achieved * * * VariableDefinition defQVariable = null; //the variable that will be read from the sdtm row as the default * PropertyDescriptor defDescriptor; //= obsDescription.DefaultPropertyDescriptor; //the matching descriptor and set it to observationDescription * * if (dataset.Domain.Class.ToLower().Equals("findings")) * { * var defVar = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESN")); * if (sdtmRow.ResultQualifiers["STRESN"] != null) * { * defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESN")); * defDescriptor = descriptorsList.Find(d => d.Name.Equals("STRES")); * } * else if (sdtmRow.ResultQualifiers["STRESC"] != null) * { * defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("STRESC")); * defDescriptor = descriptorsList.Find(d => d.Name.Equals("STRES")); * } * else //if (sdtmRow.ResultQualifiers["ORRES"] != null) * { * defQVariable = sdtmEntityDescriptor.ResultVariables.Find(rv => rv.Name.Equals("ORRES")); * defDescriptor = descriptorsList.Find(d => d.Name.Equals("ORRES")); * } * } * else if (sdtmEntityDescriptor.DomainCode.Equals("AE")) * { * defDescriptor = descriptorsList.Find(d => d.Name.Equals("OCCUR")); * //there is no defQvariable in this case as we will make it up it doesn not occur in the original dataset * } * else * { * defQVariable = sdtmEntityDescriptor.DefaultQualifier; * defDescriptor = descriptorsList.Find(d => d.Name.Equals(defQVariable.Name)); * } * * obsDescription.DefaultPropertyDescriptor = defDescriptor; * * //TODO: THIS COULD EITHER BE CVTERMS SO BOTH VARIABLE DEFINITION AND OBSERVATION DESCRIPTOR WOULD SHARE THE SAME CVTERM * //todo: CHANGE ALL VARIABLE NAMES TO ENUMS * * * if (sdtmEntityDescriptor.DomainCode.Equals("AE")) * { * CategoricalValue propValue = new CategoricalValue(); * propValue.Property = defDescriptor; * propValue.Value = "Y"; * } * else if (defDescriptor.ValueType == ObsValueType.Numerical) * { * NumericalValue propertyValue = new NumericalValue(); * propertyValue.Value = float.Parse(sdtmRow.ResultQualifiers[defQVariable.Name]); * //propertyValue.Unit = * propertyValue.Property = defDescriptor; * observation.DefaultObservedProperty = propertyValue; * } * else * { * CategoricalValue propValue = new CategoricalValue(); * propValue.Property = defDescriptor; * propValue.Value = sdtmRow.ResultQualifiers[defQVariable.Name]; * } #endregion * * /////////////////////////////////////////////////////////////////////////////////////////// * #region 3.2 Add other qualifiers * * foreach (var qualifier in sdtmRow.Qualifiers) * { * var key = qualifier.Key.Substring(2); * //GET THE MATCHING VARIABLE FROM the sdtm desriptor using names qualifier.key == * //then use its name * if (qualifier.Value != null) * { * //ADD descriptor to obsDescription if not previously added * //HOW IS THE LINK BETWEEN THE VALUE IN THE SDTM ENTITY AND THE DESCRIPTOR IS ESTABLISHED * //CURRENTLY IT IS USING THE SAME NAME FOR THE DESCRIPTOR AS THE NAME OF THE VARIABLE WITHOUT THE PREFIX * * PropertyDescriptor descriptor = obsDescription.ObservedPropertyDescriptors.Find(d => d.Name.Equals(key)); * if (descriptor == null) * { * descriptor = descriptorsList.Find(d => d.Name.Equals(key)); * obsDescription.ObservedPropertyDescriptors.Add(descriptor); * } * //ADD new observation instance * * if(descriptor.ValueType.Equals(ObsValueType.Categorical)) * //TODO: for now not checking on property value type and creating categorical value for all other than result for findings * observation.ObservedProperties.Add(new CategoricalValue() * { * Value = qualifier.Value, * Property = descriptor, * * //CVTerm = * }); * * * * } * } * #endregion * * /////////////////////////////////////////////////////////////////////////////////////////// * #region 4 Add Temporal Qualifiers * #endregion * #region 5 Add TimeSeries Descriptors * #endregion * } * } * * internal List<PropertyDescriptor> LoadObsDescriptors(SdtmRowDescriptor sdtmEntityDescriptor, int projectId) * { * * var descriptorList = new List<PropertyDescriptor>(); * * // SetDescriptorTypes(sdtmEntityDescriptor); * * //IF DOMAIN.CLASS which can be identified from sdtmEntityDescriptor * //THEN cREATE descriptor for MATCHIN CLASS * //BUT then how will this be linked with the variable name in the data file? * #region HACK * //TODO: chage this to enum * * //A HACK TO ADD OCCURENCE descriptor for adverse events which would not be in the standard * if (sdtmEntityDescriptor.DomainCode.Equals("AE")) * { * var descriptor = _ObsDescriptorRepository.FindSingle(d => d.Name.Equals("OCCUR")); * //o => o.Type == DescriptorType. * //&& o.Domain.Equals(sdtmEntityDescriptor.DomainCode)); * * if (descriptor == null) * { * descriptor = PropertyDescriptorFactory.CreateAEdefDescriptor(); * _ObsDescriptorRepository.Insert(descriptor); * } * descriptorList.Add(descriptor); * } * * //A HACK BECAUSE I DONT WANT TO HAVE TWO STANDARD RESULT DESCRIPTORS AS IT IS THE CASE WITH SDTM VARAIBLES * if (sdtmEntityDescriptor.Class.Equals("Findings")) * { * var descriptorST = _ObsDescriptorRepository.FindSingle(d => d.Name.Equals("STRES")); * if (descriptorST == null) * { * descriptorST = PropertyDescriptorFactory.CreateStandardResDescriptor(); * _ObsDescriptorRepository.Insert(descriptorST); * } * descriptorList.Add(descriptorST); * } #endregion * * foreach (var variable in sdtmEntityDescriptor.QualifierVariables.Union(sdtmEntityDescriptor.ResultVariables)) * { * //todo: SHOULD BE ABLE TO DIFFERENTIATE HERE THE DIFFERENT SUBCLASSES OF SDTM RECORD QUALIFIERS * //TODO: INTO FEATURE_QUALIFIERS, DEFAULT_OBSERVED_PROPERTY, OBSERVED_PROPERTIES, SUBJECT_QUALIFIERS AND SAMPLE_QUALIFIERS * //TODO: should find equality via CVterm not name text * //TODO: CVterm should be assigned to each variable definition which shuold then be an EQUIVALENT CVterm used for the descriptor * var descriptor = _ObsDescriptorRepository.FindSingle(o => o.Name.Equals(variable.Name.Substring(2))); * if (descriptor == null) * { * //FOR NOW the Descriptor is CREATED FROM the variable information * descriptor = new PropertyDescriptor(); * descriptor.Name = variable.Name.Substring(2); * //SOME KIND OF A METHOD HERE THAT RETURNS THE DESCRIPTOR TYPE THIS VARIABLE MATCHES * //descriptor.Type = variable.DescriptorType; //THIS IS SUPPOSED TO HAVE NOW THE NEW SUBCLASSES //DescriptorType.ObservedPropertyDescriptor; * descriptor.Description = variable.Label; * descriptor.ProjectId = projectId; * * //_ObsDescriptorRepository.Insert(descriptor); * } * descriptorList.Add(descriptor); * } * * //_dataContext.Save(); * * //TODO: consider doing the same for other qualifiers such as timings and setting variable qualifiers to qualifiers * * return descriptorList; * } */ internal bool LoadObservations(List <SdtmRow> sdtmData, SdtmRowDescriptor sdtmRowDescriptor, bool reload) { var dsDomainCode = sdtmRowDescriptor.DomainCode; var dsClass = sdtmRowDescriptor.Class; var datasetId = sdtmData.First().DatasetId; var dataFileId = sdtmData.First().DatafileId; var projectId = sdtmData.First().ProjectId; if (reload) { _ObservationRepository.DeleteMany(o => o.DatasetId == datasetId && o.DatafileId == dataFileId); _dataContext.Save().Equals("CREATED"); } //Retrieve ObjectOfObservations previously loaded for this project // CURRENTLY THIS IS NOT REALLY THE OBJECTOFOBSERVATION BUT THE OBSERVATION DESCRIPTOR WHICH IS //DEFINED PER DATASET/DATAFILE AND IS DELETABLE ACROSS LOADS AS LONG AS THE O3 CV IS UNIQUE ACROSS DATASETS //AND NOT JUST INFLUENCED BY ONE DATASET OR THE FIRST DATASET LOADED AS IT IS NOW!! var projectO3s = _ObservationRepository.FindAll(o => o.ProjectId == projectId && o.DomainCode == dsDomainCode).ToList(); List <string> O3keys = projectO3s.Select(currObservation => currObservation.Class + currObservation.DomainCode + currObservation.Group + currObservation.Name).ToList(); var observations = sdtmData.GroupBy(o => new { domain = o.DomainCode, o3 = o.Topic, group = o.Group, subgroup = o.Subgroup, o3CVterm = o.TopicControlledTerm ?? o.TopicSynonym }); foreach (var observation in observations) { var O3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3; if (O3keys.Contains(O3key)) { continue; } Observation obsDescriptor = new Observation(); obsDescriptor.Name = observation.Key.o3; obsDescriptor.Group = observation.Key.group; obsDescriptor.Subgroup = observation.Key.subgroup; obsDescriptor.Class = sdtmRowDescriptor.Class; obsDescriptor.DomainCode = sdtmRowDescriptor.DomainCode; obsDescriptor.DomainName = sdtmRowDescriptor.Domain; obsDescriptor.TopicVariable = sdtmRowDescriptor.TopicVariable; if (sdtmRowDescriptor.ObsIsAFinding) { obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.ResultVariables.Select( qualifier => new ObservationQualifier() { Observation = obsDescriptor, Qualifier = qualifier })); } //sdtmRowDescriptor.ResultVariables.Union(sdtmRowDescriptor.QualifierVariables).ToList())}; else { obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.QualifierVariables.Select( qualifier => new ObservationQualifier() { Observation = obsDescriptor, Qualifier = qualifier })); } obsDescriptor.Timings.AddRange(sdtmRowDescriptor.GetAllTimingVariables().FindAll(v => v != null).Select( qualifier => new ObservationTiming() { Observation = obsDescriptor, Qualifier = qualifier })); obsDescriptor.ControlledTermStr = observation.Key.o3CVterm; obsDescriptor.DatasetId = datasetId; obsDescriptor.DatafileId = dataFileId; obsDescriptor.ProjectId = projectId; obsDescriptor.DefaultQualifier = sdtmRowDescriptor.GetDefaultQualifier(observation.FirstOrDefault()); // //LOAD CVTERMS CORRESPONDING TO MEDdRA codes // ob = CreateObsCVterms(observation.FirstOrDefault(), sdtmRowDescriptor, out ob); // ob.ControlledTermId = observation.FirstOrDefault().QualifierQualifiers?[AECVtermIdVar]; //} _ObservationRepository.Insert(obsDescriptor); } var success = _dataContext.Save().Equals("CREATED"); //addO3IdstoSDTMrows() if (success) { //TODO: PROBLEM!! //IN CASE OF A SECOND FILE LOADED TO A PREVIOUSLY CREATED DATASET, DATAFILE WILL BE DIFFERENT //OBSERVATIONS WERE SAVED WITH THE FISRT DATASETID AND THE FIRST DATAFILE //WHAT HAPPENS WHERE WE WANT TO UNLOAD A FILE THAT BROUGHT DIFFERENT O3s to the DATASET DIFFERENT //FROM THE FIRST DATAFILE? //I STILL NEED TO BE ABLE TO FIND O3s that were loaded from a certain DATAFILE var savedObs = _ObservationRepository.FindAll(o => o.DatasetId == datasetId && o.DatafileId == dataFileId).ToList(); foreach (var observation in observations) { foreach (var sdtmRow in observation) { var O3 = savedObs.FirstOrDefault( o => o.Name == sdtmRow.Topic && o.Group == sdtmRow.Group && o.Subgroup == sdtmRow.Subgroup); if (O3 != null) { sdtmRow.DBTopicId = O3.Id; } _sdtmRepository.Update(sdtmRow); } } } return(success); }
public async Task <bool> LoadSubjects(Dataset dataset, int fileId, bool reload) { var descriptor = SdtmSubjectDescriptor.GetSdtmSubjectDescriptor(dataset); List <SdtmRow> sdtmData = await _sdtmRepository.FindAllAsync( dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId)); if (sdtmData.Count == 0) { return(false); } var projectId = sdtmData.First().ProjectId; var projectAccession = sdtmData.First().ProjectAccession; var activityId = sdtmData.First().ActivityId; var datafileId = sdtmData.First().DatafileId; if (reload) { _subjectRepository.DeleteMany(o => o.DatasetId == dataset.Id && o.DatafileId == datafileId); _dataContext.Save().Equals("CREATED"); } //Project related subject characteristics var scoList = _characteristicObjRepository.FindAll(s => s.ProjectId == projectId).ToList(); var scos = scoList.ToDictionary(co => co.ShortName); //Project related studies var studies = _studtRepository.FindAll(s => s.ProjectId == projectId, new List <string> { "Arms.Arm" }).ToList(); var studyMap = studies.ToDictionary(study => study.Name); //Project related arms var arms = studies.SelectMany(s => s.Arms.Select(a => a.Arm)).Distinct(); var armMap = arms.ToDictionary(arm => arm.Name); //Previously loaded subjects for this project that might be update by this load //TODO: should I add datafileId to subject as well? to be able to delete subjects for this datafile var subjects = _subjectRepository.FindAll(s => s.Study.ProjectId == projectId).ToList(); foreach (var sdtmSubject in sdtmData) { Study study; if (!studyMap.TryGetValue(sdtmSubject.StudyId, out study)) { study = new Study() { Name = sdtmSubject.StudyId, Accession = "S-" + projectAccession.Substring(2) + "-" + (studies.Count() + 1).ToString("00"), ProjectId = projectId }; studyMap.Add(study.Name, study); } Arm arm; if (!armMap.TryGetValue(sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name], out arm)) { arm = new Arm() { Id = "P-" + projectId + "-ARM-" + sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name], Code = sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name], Name = sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name] }; armMap.Add(arm.Name, arm); } if (!arm.Studies.Exists(s => s.Study.Name == study.Name)) { arm.Studies.Add(new StudyArm() { Arm = arm, Study = study }); } var subjNewFlag = false; /** * ADDING SUBJECTS */ var subject = subjects.Find(s => s.UniqueSubjectId == sdtmSubject.USubjId); if (subject == null) { subject = new HumanSubject { Id = "P-" + projectId + "-" + sdtmSubject.USubjId, UniqueSubjectId = sdtmSubject.USubjId, SubjectStudyId = sdtmSubject.SubjectId, Arm = sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name], //Will put in characteristics for now ArmCode = sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name], DatasetId = sdtmSubject.DatasetId, DatafileId = sdtmSubject.DatafileId, Study = study, StudyArm = arm }; subjNewFlag = true; } else { subjNewFlag = false; subject.DatasetId = dataset.Id; subject.DatafileId = sdtmSubject.DatafileId; subject.Arm = sdtmSubject.QualifierSynonyms[descriptor.ArmVariable.Name]; subject.ArmCode = sdtmSubject.Qualifiers[descriptor.ArmCodeVariable.Name]; subject.Study = study; subject.StudyArm = arm; } //SET/UPDATE SUBJECT CHARACTERISTICS if (descriptor.RefStartDate != null && sdtmSubject.Qualifiers.ContainsKey(descriptor?.RefStartDate?.Name)) { DateTime startDate; if (DateTime.TryParse(sdtmSubject.Qualifiers[descriptor.RefStartDate?.Name], out startDate)) { subject.SubjectStartDate = startDate; } } if (descriptor.RefEndDate != null && sdtmSubject.Qualifiers.ContainsKey(descriptor?.RefEndDate?.Name)) { DateTime endDate; if (DateTime.TryParse(sdtmSubject.Qualifiers[descriptor.RefEndDate.Name], out endDate)) { subject.SubjectEndDate = endDate; } } //THIS METHOD WILL DELETE ALL SUBJECT CHARACTERISTICS PREVIOUSLY LOADED //FOR THIS SUBJECT FOR THE SAME DATAFILE //((List<SubjectCharacteristic>)subject.SubjectCharacteristics) // .RemoveAll(sc=>sc.DatafileId == datafileId && sc.DatasetId == datasetId); //_subjectRepository.Update(subject); //_dataContext.Save(); /** * ADDING SUBJECT CHARACTERISTICS */ foreach (var qualifier in sdtmSubject.Qualifiers) { //THIS CHECKS THAT ONLY COLUMNS IN DATA THAT ARE PRESENT IN THE TEMPLATE ARE PARSED var dsVar = descriptor.QualifierVariables.SingleOrDefault(v => v.Name.Equals(qualifier.Key)); if (descriptor.CharacteristicProperties.Exists(c => c.Name == qualifier.Key) || descriptor.TimingVariableDefinitions.Exists(c => c.Name == qualifier.Key)) { CharacteristicFeature sco; scos.TryGetValue(qualifier.Key, out sco); if (sco == null) { /** * CREATING NEW CHARACTERISTIC OBJECT */ sco = new CharacteristicFeature() { ShortName = dsVar.Name, FullName = dsVar.Label, Domain = "DM", ProjectId = projectId, DataType = dsVar.DataType, ActivityId = activityId }; scos.Add(dsVar.Name, sco); } /** * ADDING SUBJECT CHARACTERISTIC */ subject.SubjectCharacteristics.Add(new SubjectCharacteristic() { CharacteristicFeature = sco, VerbatimValue = qualifier.Value, VerbatimName = sco.ShortName, Subject = subject, DatasetId = dataset.Id, DatafileId = datafileId }); } } if (subjNewFlag) { _subjectRepository.Insert(subject); } else { _subjectRepository.Update(subject); } } return(_dataContext.Save().Equals("CREATED")); }
public bool LoadBioSamples(List <SdtmRow> sampleData, int datasetId) { var dataset = _datasetRepository.FindSingle(d => d.Id.Equals(datasetId), new List <string>() { "Variables.VariableDefinition" }); var studyMap = new Dictionary <string, int>(); var scoList = _characteristicObjRepository.FindAll(s => s.ProjectId.Equals(dataset.Activity.ProjectId)).ToList(); var scos = scoList.ToDictionary(co => co.ShortName); foreach (SdtmRow sdtmEntity in sampleData) { Study study; int studyid; if (!studyMap.TryGetValue(sdtmEntity.StudyId, out studyid)) { study = _studyRepository.FindSingle(s => s.Name.Equals(sdtmEntity.StudyId)); studyMap.Add(sdtmEntity.StudyId, study.Id); studyid = study.Id; } /** * ADDING BIOSAMPLE */ var bioSample = new Biosample() { BiosampleStudyId = sdtmEntity.SampleId, AssayId = sdtmEntity.ActivityId, SubjectId = sdtmEntity.USubjId, StudyId = studyid, //Visit = sdtmEntity.Visit, //CollectionStudyDay = sdtmEntity.CollectionStudyDay, DatasetId = sdtmEntity.DatasetId, IsBaseline = sdtmEntity.BaseLineFlag }; /** * ADDING SAMPLE CHARACTERISTICS */ foreach (var resqualifier in sdtmEntity.ResultQualifiers) { var dsVar = dataset.Variables.SingleOrDefault(v => v.VariableDefinition.Name.Equals(resqualifier.Key)); if (dsVar != null) { CharacteristicFeature sco; scos.TryGetValue(sdtmEntity.Topic, out sco); if (sco == null) { /** * CREATING NEW CHARACTERISTIC OBJECT */ sco = new CharacteristicFeature() { ShortName = sdtmEntity.Topic, //characteristicVar.VariableDefinition.Name, FullName = sdtmEntity.TopicSynonym, //characteristicVar.VariableDefinition.Label, Domain = "BS", ProjectId = dataset.Activity.ProjectId, //DataType = dsVar.VariableDefinition.DataType }; scos.Add(sdtmEntity.Topic, sco); } bioSample.SampleCharacteristics.Add(new SampleCharacteristic() { // DatasetVariable = dsVar, CharacteristicFeature = sco, VerbatimName = sdtmEntity.TopicSynonym, VerbatimValue = resqualifier.Value, //DatasetDomainCode = sdtmEntity.DomainCode }); } } foreach (var qualifier in sdtmEntity.Qualifiers) { var dsVar = dataset.Variables.SingleOrDefault(v => v.VariableDefinition.Name.Equals(qualifier.Key)); if (dsVar != null) { CharacteristicFeature sco; scos.TryGetValue(qualifier.Key, out sco); if (sco == null) { /** * CREATING NEW CHARACTERISTIC OBJECT */ sco = new CharacteristicFeature() { ShortName = dsVar.VariableDefinition.Name, FullName = dsVar.VariableDefinition.Label, Domain = "BS", ProjectId = dataset.Activity.ProjectId, //DataType = dsVar.VariableDefinition.DataType }; scos.Add(dsVar.VariableDefinition.Name, sco); } bioSample.SampleCharacteristics.Add(new SampleCharacteristic() { VerbatimName = qualifier.Key, VerbatimValue = qualifier.Value, //DatasetDomainCode = sdtmEntity.DomainCode, //DatasetVariable =dsVar, CharacteristicFeature = sco, }); } } _bioSampleRepository.Insert(bioSample); } return(_dataContext.Save().Equals("CREATED")); }
private UserDataset CreateOrFindSubjectClinicalDataset(CombinedQuery query, string userId) { var phenoDataset = _userDatasetRepository.FindSingle( d => d.ProjectId == query.ProjectId && d.QueryId == query.Id && d.Type == "PHENO" && d.OwnerId == userId); if (phenoDataset != null) { return(phenoDataset); } phenoDataset = new UserDataset { Id = Guid.NewGuid(), OwnerId = userId, ProjectId = query.ProjectId, Type = "PHENO", Name = "Subjects", QueryId = query.Id, FileStatus = 0, }; //ADD SUBJECTID & STUDYID DATAFIELD phenoDataset.Fields.Add(CreateSubjectIdField()); phenoDataset.Fields.Add(CreateStudyIdField()); //ADD DESIGN ELEMENT FIELDS (STUDY, VISIT, ARM...etc) phenoDataset.Fields.AddRange(query.DesignElements.Select(qObj => new DatasetField() { QueryObject = qObj, QueryObjectType = qObj.QueryFor, ColumnHeader = qObj.QueryObjectName })); //ADD SUBJECT CHARACTERISTICS (AGE, RACE, SEX ...etc) phenoDataset.Fields.AddRange(query.SubjectCharacteristics.Select(qObj => new DatasetField() { QueryObject = qObj, QueryObjectType = nameof(SubjectCharacteristic), ColumnHeader = qObj.QueryObjectName//.ObservationName })); //ADD CLINICAL OBSERVATIONS foreach (var co in query.ClinicalObservations) { phenoDataset.Fields.Add(new DatasetField() { QueryObject = co, QueryObjectType = nameof(SdtmRow), ColumnHeader = co.ObservationName }); if (co.HasLongitudinalData) { phenoDataset.Fields.Add(new DatasetField() { QueryObject = new Query() { QueryFor = nameof(Visit), QueryFrom = nameof(SdtmRow), QuerySelectProperty = "Name" }, QueryObjectType = nameof(Visit), ColumnHeader = "visit" }); } if (co.HasTPT) { phenoDataset.Fields.Add(new DatasetField() { QueryObject = new Query() { QueryFor = nameof(SdtmRow.CollectionStudyTimePoint), QueryFrom = nameof(SdtmRow), QuerySelectProperty = "Name" }, QueryObjectType = nameof(RelativeTimePoint), ColumnHeader = "timepoint" }); } } //phenoDataset.Fields.AddRange(query.ClinicalObservations.Select(qObj => new DatasetField() //{ // QueryObject = qObj, // QueryObjectType = nameof(SdtmRow), // ColumnHeader = qObj.ObservationName //})); //ADD GROUPED CLINICAL OBSERVATIONS phenoDataset.Fields.AddRange(query.GroupedObservations.Select(gObs => new DatasetField() { QueryObject = gObs, QueryObjectType = nameof(SdtmRow), ColumnHeader = gObs.ObservationName })); var exportData = _queryService.GetQueryResult(query.Id); phenoDataset.SubjectCount = exportData.Subjects.Count; _userDatasetRepository.Insert(phenoDataset); _dataContext.Save(); return(phenoDataset); }
public async Task <bool> LoadObservations(Dataset dataset, int fileId, bool reload) { var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(dataset); List <SdtmRow> sdtmData = await _sdtmRepository.FindAllAsync( dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId)); var dsDomainCode = sdtmRowDescriptor.DomainCode; var dsClass = sdtmRowDescriptor.Class; var datasetId = sdtmData.First().DatasetId; var dataFileId = sdtmData.First().DatafileId; var projectId = sdtmData.First().ProjectId; if (reload) { _observationRepository.DeleteMany(o => o.DatasetId == datasetId && o.DatafileId == dataFileId); _dataContext.Save(); } //Retrieve ObjectOfObservations previously loaded for this project // CURRENTLY THIS IS NOT REALLY THE OBJECTOFOBSERVATION BUT THE OBSERVATION DESCRIPTOR WHICH IS //DEFINED PER DATASET/DATAFILE AND IS DELETABLE ACROSS LOADS AS LONG AS THE O3 CV IS UNIQUE ACROSS DATASETS //AND NOT JUST INFLUENCED BY ONE DATASET OR THE FIRST DATASET LOADED AS IT IS NOW!! var projectO3s = _observationRepository.FindAll(o => o.ProjectId == projectId && o.DomainCode == dsDomainCode).ToList(); //List<string> O3keys = // projectO3s.Select( // currObservation => // currObservation.Class + currObservation.DomainCode + currObservation.Group + // currObservation.Name).ToList(); var O3map = projectO3s.ToDictionary(o3 => o3.Class + o3.DomainCode + o3.Group + o3.Name + o3.ControlledTermStr); var observations = sdtmData.GroupBy( o => new { domain = o.DomainCode, o3 = o.Topic, group = o.Group, subgroup = o.Subgroup, o3CVterm = o.TopicControlledTerm ?? o.TopicSynonym }); var obsPrevLoaded = new Dictionary <string, Observation>(); foreach (var observation in observations) { var O3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3 + observation.Key.o3CVterm; Observation o3; if (O3map.TryGetValue(O3key, out o3)) { obsPrevLoaded.Add(O3key, o3); continue; } Observation obsDescriptor = new Observation(); obsDescriptor.Name = observation.Key.o3; obsDescriptor.Group = observation.Key.group; obsDescriptor.Subgroup = observation.Key.subgroup; obsDescriptor.Class = sdtmRowDescriptor.Class; obsDescriptor.DomainCode = sdtmRowDescriptor.DomainCode; obsDescriptor.DomainName = sdtmRowDescriptor.Domain; obsDescriptor.TopicVariable = sdtmRowDescriptor.TopicVariable; if (sdtmRowDescriptor.ObsIsAFinding) { obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.ResultVariables.Select( qualifier => new ObservationQualifier() { Observation = obsDescriptor, Qualifier = qualifier })); } //sdtmRowDescriptor.ResultVariables.Union(sdtmRowDescriptor.QualifierVariables).ToList())}; obsDescriptor.Qualifiers.AddRange(sdtmRowDescriptor.QualifierVariables.Select( qualifier => new ObservationQualifier() { Observation = obsDescriptor, Qualifier = qualifier })); obsDescriptor.Timings.AddRange(sdtmRowDescriptor.TimeVariables.Select( qualifier => new ObservationTiming() { Observation = obsDescriptor, Qualifier = qualifier })); obsDescriptor.ControlledTermStr = observation.Key.o3CVterm; obsDescriptor.DatasetId = datasetId; obsDescriptor.DatafileId = dataFileId; obsDescriptor.ProjectId = projectId; obsDescriptor.DefaultQualifier = sdtmRowDescriptor.GetDefaultQualifier(observation.FirstOrDefault()); var newObs = _observationRepository.Insert(obsDescriptor); obsPrevLoaded.Add(O3key, newObs); } var success = _dataContext.Save().Equals("CREATED"); if (success) { //TODO: PROBLEM!! //IN CASE OF A SECOND FILE LOADED TO A PREVIOUSLY CREATED DATASET, DATAFILE WILL BE DIFFERENT //OBSERVATIONS WERE SAVED WITH THE FISRT DATASETID AND THE FIRST DATAFILE //WHAT HAPPENS WHERE WE WANT TO UNLOAD A FILE THAT BROUGHT DIFFERENT O3s to the DATASET DIFFERENT //FROM THE FIRST DATAFILE? //I STILL NEED TO BE ABLE TO FIND O3s that were loaded from a certain DATAFILE var savedObs = _observationRepository.FindAll(o => o.DatasetId == datasetId && o.DatafileId == dataFileId).ToList(); foreach (var observation in observations) { foreach (var sdtmRow in observation) { Observation O3; var o3key = dsClass + observation.Key.domain + observation.Key.group + observation.Key.o3 + observation.Key.o3CVterm; if (obsPrevLoaded.TryGetValue(o3key, out O3)) { sdtmRow.DBTopicId = O3.Id; } _sdtmRepository.Update(sdtmRow); } } } return(success); }
public async Task <bool> LoadDataset(int datasetId, int fileId) { Dataset dataset = GetDatasetDescriptor(datasetId); var reload = false; var loaded = false; var dataFile = _dataFileRepository.Get(fileId); if (dataFile.IsLoadedToDB) { reload = true; } try { switch (dataset.Template.Code) { case "DM": var subjectLoader = new SubjectLoader(_dataServiceUnit); loaded = await subjectLoader.LoadSubjects(dataset, fileId, reload); break; case "BS": var sampleLoader = new BioSampleLoader(_dataServiceUnit); loaded = await sampleLoader.LoadBioSamples(dataset, fileId, reload); break; case "CY": case "HD": var hdDataLoader = new HDloader(_dataServiceUnit); //loaded = await hdDataLoader.LoadHDdata(dataset,fileId, datasetId); break; case "MX": break; default: var observationLoader = new ObservationLoader(this._dataServiceUnit); loaded = await observationLoader.LoadObservations(dataset, fileId, reload); if (loaded) { await _cacheService.GenerateAndCacheClinicalDTO(dataFile.ProjectId); } break; } if (loaded) { dataFile.State = "LOADED"; dataFile.IsLoadedToDB = true; _dataFileRepository.Update(dataFile); if (!dataset.DataFiles.Select(d => d.DatafileId).Contains(fileId)) { //Adding a new datafile to this dataset dataset.DataFiles.Add(new DatasetDatafile() { DatasetId = datasetId, DatafileId = dataFile.Id }); _datasetRepository.Update(dataset); } _dataServiceUnit.Save(); } if (!loaded && dataFile.State == "SAVED") { _sdtmRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId); } } catch (Exception e) { Debug.WriteLine("Failed to load descriptors to SQL database", e.Message); await UnloadDataset(datasetId, fileId, "LOADING FAILED"); } return(loaded); }
public async Task <bool> LoadBioSamples(Dataset dataset, int fileId, bool reload) { var descriptor = SdtmSampleDescriptor.GetSdtmSampleDescriptor(dataset); List <SdtmRow> sampleData = await _sdtmRepository.FindAllAsync( dm => dm.DatasetId.Equals(dataset.Id) && dm.DatafileId.Equals(fileId)); if (sampleData.Count == 0) { return(false); } var projectId = sampleData.First().ProjectId; var projectAccession = sampleData.First().ProjectAccession; var assayId = sampleData.First().ActivityId; var datasetId = sampleData.First().DatasetId; var datafileId = sampleData.First().DatafileId; if (projectId == 0 || assayId == 0) { return(false); } var assay = _assayRepository.Get(assayId); if (reload) { _bioSampleRepository.DeleteMany(o => o.DatasetId == datasetId && o.DataFileId == datafileId); _dataContext.Save().Equals("CREATED"); } var studyMap = new Dictionary <string, int>(); //RETRIEVE PREVIOUSLY LOADED CHARACTERISTIC FEATURES FOR THIS ASSAY THAT MIGHT HAVE BEEN LOADED BEFORE FOR OTHER STUDIES var featureList = _characteristicObjRepository.FindAll(s => s.ProjectId == projectId && s.ActivityId == assayId).ToList(); _featureMap = featureList.ToDictionary(co => co.ShortName); var subjects = _subjectRepository.FindAll(s => s.Study.ProjectId == projectId).ToList(); var sampleDayMap = new Dictionary <string, int>(); foreach (var sdtmRow in sampleData) { //RETRIEVE STUDYID FROM DB AND STORE IN MAP LOCALLY TO AVOID MAKING TOO MANY DB CALLS int studyid; if (!studyMap.TryGetValue(sdtmRow.StudyId, out studyid)) { var study = _studyRepository.FindSingle(s => s.Name == sdtmRow.StudyId && s.ProjectId == projectId); studyMap.Add(sdtmRow.StudyId, study.Id); studyid = study.Id; } /** * ADDING BIOSAMPLE */ var bioSample = new Biosample() { BiosampleStudyId = sdtmRow.SampleId, AssayId = sdtmRow.ActivityId, SubjectId = subjects.Find(s => s.UniqueSubjectId == sdtmRow.USubjId)?.Id, StudyId = studyid, CollectionStudyDay = sdtmRow.CollectionStudyDay, DatasetId = sdtmRow.DatasetId, DataFileId = sdtmRow.DatafileId, IsBaseline = sdtmRow.Qualifiers != null && descriptor.IsBaseLineVariable != null && (sdtmRow.Qualifiers.ContainsKey(descriptor.IsBaseLineVariable?.Name) && sdtmRow.Qualifiers[descriptor.IsBaseLineVariable?.Name] == "Y") }; //CHECK IF ASSAY HAS TEMOPORALDATA if (bioSample.CollectionStudyDay?.Number != null && !assay.HasTemporalData) { if (sampleDayMap.ContainsKey(bioSample.SubjectId) && sampleDayMap[bioSample.SubjectId] != bioSample.CollectionStudyDay.Number.Value && !assay.HasTemporalData) { assay.HasTemporalData = true; } else { if (!sampleDayMap.ContainsKey(bioSample.SubjectId)) { sampleDayMap.Add(bioSample.SubjectId, bioSample.CollectionStudyDay.Number.Value); } } } //ADD CHARACTERISTIC FEATURE IN BSTEST string charValue; var characValueVariable = descriptor.GetValueVariable(sdtmRow); if (characValueVariable != null && sdtmRow.ResultQualifiers.TryGetValue(characValueVariable.Name, out charValue)) { var charFeature = getCharacteristicFeature(sdtmRow.Topic, sdtmRow.TopicSynonym, characValueVariable.DataType); charFeature.ActivityId = assayId; charFeature.ProjectId = projectId; bioSample.SampleCharacteristics.Add(new SampleCharacteristic() { CharacteristicFeature = charFeature, VerbatimName = sdtmRow.TopicSynonym, VerbatimValue = charValue, DatafileId = datafileId, DatasetId = datasetId }); } //ADDING SPECIMEN TYPE TO CHARACTERISTICS string st; if (descriptor.SpecimenTypeVariable != null && sdtmRow.Qualifiers.TryGetValue(descriptor.SpecimenTypeVariable.Name, out st)) { var feature = getCharacteristicFeature(descriptor.SpecimenTypeVariable.Name, descriptor.SpecimenTypeVariable.Label, descriptor.SpecimenTypeVariable.DataType); feature.ActivityId = assayId; feature.ProjectId = projectId; bioSample.SampleCharacteristics.Add(new SampleCharacteristic() { CharacteristicFeature = feature, VerbatimName = descriptor.SpecimenTypeVariable?.Name, VerbatimValue = st, DatafileId = datafileId, DatasetId = datasetId }); } //ADD ANATOMIC REGION TO CHARACTERISTICS if (descriptor.AnatomicRegionVariable != null && sdtmRow.QualifierQualifiers.TryGetValue(descriptor.AnatomicRegionVariable.Name, out st)) { var feature = getCharacteristicFeature(descriptor.AnatomicRegionVariable.Name, descriptor.AnatomicRegionVariable.Label, descriptor.AnatomicRegionVariable.DataType); feature.ActivityId = assayId; feature.ProjectId = projectId; bioSample.SampleCharacteristics.Add(new SampleCharacteristic() { CharacteristicFeature = feature, VerbatimName = descriptor.AnatomicRegionVariable.Name, VerbatimValue = st, DatafileId = datafileId, DatasetId = datasetId }); } _bioSampleRepository.Insert(bioSample); _assayRepository.Update(assay); } return(_dataContext.Save().Equals("CREATED")); }
public bool LoadSDTM(int datasetId, int fileId, DataTable dataTable) { var dataset = _datasetRepository.FindSingle(d => d.Id == datasetId, new List <string>() { "Variables.VariableDefinition", "Template", "Activity.Project" }); var dataFile = _dataFileRepository.Get(fileId); if (!dataFile.State.ToLower().Equals("new")) { /** * Replacing previously loaded file * Remove file from collection before reloading it */ _sdtmRepository.DeleteMany(s => s.DatafileId == fileId && s.DatasetId == datasetId); Debug.WriteLine("RECORD(s) SUCCESSFULLY DELETED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId); } var sdtmRowDescriptor = SdtmRowDescriptor.GetSdtmRowDescriptor(dataset); var SDTM = new List <SdtmRow>(); var totalLoaded = 0.0; try { foreach (DataRow row in dataTable.Rows) { var sdtmRow = SDTMreader.readSDTMrow(row, dataTable, sdtmRowDescriptor); sdtmRow.Id = Guid.NewGuid(); sdtmRow.DatasetId = datasetId; sdtmRow.ActivityId = dataset.ActivityId; sdtmRow.DatafileId = fileId; sdtmRow.ProjectId = dataset.Activity.ProjectId; sdtmRow.ProjectAccession = dataset.Activity.Project.Accession; SDTM.Add(sdtmRow); if (SDTM.Count % 100 == 0) { _sdtmRepository.InsertMany(SDTM); totalLoaded += SDTM.Count; dataFile.State = Math.Round(totalLoaded / dataTable.Rows.Count * 100).ToString("##"); _dataFileRepository.Update(dataFile); _dataServiceUnit.Save(); SDTM.Clear(); } } _sdtmRepository.InsertMany(SDTM); Debug.WriteLine(dataTable.Rows.Count + " RECORD(s) SUCCESSFULLY ADDED FOR DATASET:" + datasetId + " ,DATAFILE:" + fileId); } catch (Exception e) { dataFile.State = "FAILED"; dataFile.IsLoadedToDB = false; _dataFileRepository.Update(dataFile); _dataServiceUnit.Save(); Debug.WriteLine(e.Message); return(false); } dataFile.State = "SAVED"; _dataFileRepository.Update(dataFile); _dataServiceUnit.Save(); return(true); }