public ActionResult Index( long[] matterIds, short characteristicLinkId, short subsequencesCharacteristicLinkId, Feature[] features, string maxPercentageDifference, string[] filters) { return(CreateTask(() => { double percentageDifference = double.Parse(maxPercentageDifference, CultureInfo.InvariantCulture) / 100; var attributeValuesCache = new AttributeValueCacheManager(); var characteristics = new SubsequenceData[matterIds.Length][]; long[] parentSequenceIds; var matterNames = new string[matterIds.Length]; int mattersCount = matterIds.Length; Dictionary <string, object> characteristicsTypesData; using (var db = new LibiadaWebEntities()) { // Sequences characteristic var geneticSequenceRepository = new GeneticSequenceRepository(db); long[] chains = geneticSequenceRepository.GetNucleotideSequenceIds(matterIds); // Sequences characteristic matterIds = OrderMatterIds(matterIds, characteristicLinkId, chains); // Subsequences characteristics var parentSequences = db.DnaSequence.Include(s => s.Matter) .Where(s => s.Notation == Notation.Nucleotides && matterIds.Contains(s.MatterId)) .Select(s => new { s.Id, s.MatterId, MatterName = s.Matter.Name }) .ToDictionary(s => s.Id); parentSequenceIds = parentSequences .OrderBy(ps => Array.IndexOf(matterIds, ps.Value.MatterId)) .Select(ps => ps.Key) .ToArray(); for (int n = 0; n < parentSequenceIds.Length; n++) { matterNames[n] = parentSequences[parentSequenceIds[n]].MatterName; } var viewDataHelper = new ViewDataHelper(db); characteristicsTypesData = viewDataHelper.GetCharacteristicsData(CharacteristicCategory.Full); } FullCharacteristicRepository fullCharacteristicRepository = FullCharacteristicRepository.Instance; string sequenceCharacteristicName = fullCharacteristicRepository.GetCharacteristicName(characteristicLinkId); string characteristicName = fullCharacteristicRepository.GetCharacteristicName(subsequencesCharacteristicLinkId); var characteristicValueSubsequences = new Dictionary <double, List <(int matterIndex, int subsequenceIndex)> >(); // cycle through matters for (int i = 0; i < mattersCount; i++) { SubsequenceData[] subsequencesData = CalculateSubsequencesCharacteristics( new[] { subsequencesCharacteristicLinkId }, features, parentSequenceIds[i], filters); characteristics[i] = subsequencesData; attributeValuesCache.FillAttributeValues(subsequencesData); for (int j = 0; j < subsequencesData.Length; j++) { double value = subsequencesData[j].CharacteristicsValues[0]; if (characteristicValueSubsequences.TryGetValue(value, out List <(int matterIndex, int subsequenceIndex)> matterAndSubsequenceIdsList)) { matterAndSubsequenceIdsList.Add((i, j)); }
public ActionResult Index( string searchQuery, bool importGenes, bool importPartial, bool filterMinLength, int minLength, bool filterMaxLength, int maxLength) { return(CreateTask(() => { string searchResults; string[] accessions; List <NuccoreObject> nuccoreObjects; if (filterMinLength) { searchResults = filterMaxLength ? NcbiHelper.FormatNcbiSearchTerm(searchQuery, minLength, maxLength: maxLength) : NcbiHelper.FormatNcbiSearchTerm(searchQuery, minLength); } else { searchResults = filterMaxLength ? NcbiHelper.FormatNcbiSearchTerm(searchQuery, minLength: 1, maxLength: maxLength) : NcbiHelper.FormatNcbiSearchTerm(searchQuery); } nuccoreObjects = NcbiHelper.ExecuteESummaryRequest(searchResults, importPartial); accessions = nuccoreObjects.Select(no => no.AccessionVersion.Split('.')[0]).Distinct().ToArray(); var importResults = new List <MatterImportResult>(accessions.Length); using (var db = new LibiadaWebEntities()) { var matterRepository = new MatterRepository(db); var dnaSequenceRepository = new GeneticSequenceRepository(db); var(existingAccessions, accessionsToImport) = dnaSequenceRepository.SplitAccessionsIntoExistingAndNotImported(accessions); importResults.AddRange(existingAccessions.ConvertAll(existingAccession => new MatterImportResult { MatterName = existingAccession, Result = "Sequence already exists", Status = "Exists" })); foreach (string accession in accessionsToImport) { var importResult = new MatterImportResult() { MatterName = accession }; try { ISequence bioSequence = NcbiHelper.DownloadGenBankSequence(accession); GenBankMetadata metadata = NcbiHelper.GetMetadata(bioSequence); importResult.MatterName = metadata.Version.CompoundAccession; Matter matter = matterRepository.CreateMatterFromGenBankMetadata(metadata); importResult.SequenceType = matter.SequenceType.GetDisplayValue(); importResult.Group = matter.Group.GetDisplayValue(); importResult.MatterName = matter.Name; importResult.AllNames = $"Common name = {metadata.Source.CommonName}, " + $"Species = {metadata.Source.Organism.Species}, " + $"Definition = {metadata.Definition}, " + $"Saved matter name = {importResult.MatterName}"; var sequence = new CommonSequence { Matter = matter, Notation = Notation.Nucleotides, RemoteDb = RemoteDb.GenBank, RemoteId = metadata.Version.CompoundAccession }; bool partial = metadata.Definition.ToLower().Contains("partial"); dnaSequenceRepository.Create(sequence, bioSequence, partial); (importResult.Result, importResult.Status) = importGenes ? ImportFeatures(metadata, sequence) : ("Successfully imported sequence", "Success"); } catch (Exception exception) { importResult.Status = "Error"; importResult.Result = $"Error: {exception.Message}"; while (exception.InnerException != null) { exception = exception.InnerException; importResult.Result += $" {exception.Message}"; } foreach (var dbEntityEntry in db.ChangeTracker.Entries()) { if (dbEntityEntry.Entity != null) { dbEntityEntry.State = EntityState.Detached; } } } finally { importResults.Add(importResult); } } string[] names = importResults.Select(r => r.MatterName).ToArray(); // removing matters for which adding of sequence failed Matter[] orphanMatters = db.Matter .Include(m => m.Sequence) .Where(m => names.Contains(m.Name) && m.Sequence.Count == 0) .ToArray(); if (orphanMatters.Length > 0) { db.Matter.RemoveRange(orphanMatters); db.SaveChanges(); } } var result = new Dictionary <string, object> { { "result", importResults } }; return new Dictionary <string, string> { { "data", JsonConvert.SerializeObject(result) } }; })); }
public ActionResult Index(bool reinmportSequences) { return(CreateTask(() => { if (reinmportSequences) { throw new NotImplementedException(); } Dictionary <string, AccessionUpdateSearchResult> sequencesData = new Dictionary <string, AccessionUpdateSearchResult>(); using (var db = new LibiadaWebEntities()) { var dnaSequenceRepository = new GeneticSequenceRepository(db); var sequencesWithAccessions = db.DnaSequence .Include(ds => ds.Matter) .Where(ds => ds.Notation == Notation.Nucleotides && !string.IsNullOrEmpty(ds.RemoteId)) .ToArray(); sequencesData = sequencesWithAccessions .ToDictionary(s => s.RemoteId.Split('.')[0], s => new AccessionUpdateSearchResult() { LocalAccession = s.RemoteId, LocalVersion = Convert.ToByte(s.RemoteId.Split('?')[0].Split('.')[1]), Name = s.Matter.Name.Split('|')[0].Trim(), LocalUpdateDate = s.Matter.Modified.ToString(OutputFormats.DateFormat), LocalUpdateDateTime = s.Matter.Modified }); } List <NuccoreObject> searchResults = new List <NuccoreObject>(); // slicing accessions into chunks to prevent "too long request" error string[] accessions = sequencesData.Keys.ToArray(); const int maxChunkSize = 10000; for (int i = 0; i < accessions.Length; i += maxChunkSize) { int actualChunkSize = Math.Min(maxChunkSize, accessions.Length - i); var accessionsChunk = new string[actualChunkSize]; Array.Copy(accessions, i, accessionsChunk, 0, actualChunkSize); (string ncbiWebEnvironment, string queryKey) = NcbiHelper.ExecuteEPostRequest(string.Join(",", accessionsChunk)); searchResults.AddRange(NcbiHelper.ExecuteESummaryRequest(ncbiWebEnvironment, queryKey, true)); } for (int i = 0; i < searchResults.Count; i++) { var searchResult = searchResults[i]; searchResult.Title = searchResult.Title.TrimEnd(".") .TrimEnd(", complete genome") .TrimEnd(", complete sequence") .TrimEnd(", complete CDS") .TrimEnd(", complete cds") .TrimEnd(", genome"); var newAccession = searchResult.AccessionVersion.Split('.'); var sequenceData = sequencesData[newAccession[0]]; sequenceData.RemoteVersion = Convert.ToByte(newAccession[1]); sequenceData.RemoteName = searchResult.Title; sequenceData.RemoteOrganism = searchResult.Organism; sequenceData.RemoteUpdateDate = searchResult.UpdateDate.ToString(OutputFormats.DateFormat); sequenceData.Updated = sequenceData.LocalUpdateDateTime <= searchResult.UpdateDate || sequenceData.RemoteVersion > sequenceData.LocalVersion; sequenceData.NameUpdated = !(sequenceData.Name.Contains(searchResult.Title) && sequenceData.Name.Contains(searchResult.Organism)); } var result = new Dictionary <string, object> { { "results", sequencesData.Values .OrderByDescending(r => r.RemoteVersion - r.LocalVersion) .ThenBy(r => r.Updated) .ThenBy(r => r.NameUpdated) } }; return new Dictionary <string, string> { { "data", JsonConvert.SerializeObject(result) } }; })); }
public ActionResult Create( [Bind(Include = "Id,Notation,RemoteDb,RemoteId,Description,Matter,MatterId")] CommonSequence commonSequence, bool localFile, Language?language, bool?original, Translator?translator, bool?partial, int?precision) { return(CreateTask(() => { var db = new LibiadaWebEntities(); try { if (!ModelState.IsValid) { throw new Exception("Model state is invalid"); } Stream sequenceStream; Nature nature = commonSequence.Notation.GetNature(); if (nature == Nature.Genetic && !localFile) { sequenceStream = NcbiHelper.GetFastaFileStream(commonSequence.RemoteId); } else { sequenceStream = FileHelper.GetFileStream(Request.Files[0]); } switch (nature) { case Nature.Genetic: ISequence bioSequence = NcbiHelper.GetFastaSequence(sequenceStream); var dnaSequenceRepository = new GeneticSequenceRepository(db); dnaSequenceRepository.Create(commonSequence, bioSequence, partial ?? false); break; case Nature.Music: var musicSequenceRepository = new MusicSequenceRepository(db); musicSequenceRepository.Create(commonSequence, sequenceStream); break; case Nature.Literature: var literatureSequenceRepository = new LiteratureSequenceRepository(db); literatureSequenceRepository.Create(commonSequence, sequenceStream, language ?? Language.Russian, original ?? true, translator ?? Translator.NoneOrManual); break; case Nature.MeasurementData: var dataSequenceRepository = new DataSequenceRepository(db); dataSequenceRepository.Create(commonSequence, sequenceStream, precision ?? 0); break; case Nature.Image: var matterRepository = new MatterRepository(db); int fileSize = Request.Files[0].ContentLength; var file = new byte[fileSize]; Request.Files[0].InputStream.Read(file, 0, fileSize); var matter = new Matter { Nature = Nature.Image, SequenceType = commonSequence.Matter.SequenceType, Name = commonSequence.Matter.Name, Source = file, Group = commonSequence.Matter.Group }; matterRepository.SaveToDatabase(matter); break; default: throw new InvalidEnumArgumentException(nameof(nature), (int)nature, typeof(Nature)); } string multisequenceName = db.Multisequence.SingleOrDefault(ms => ms.Id == commonSequence.Matter.MultisequenceId).Name; var result = new ImportResult(commonSequence, language, original, translator, partial, precision, multisequenceName); return new Dictionary <string, string> { { "data", JsonConvert.SerializeObject(result) } }; } catch (Exception) { long matterId = commonSequence.MatterId; if (matterId != 0) { List <Matter> orphanMatter = db.Matter .Include(m => m.Sequence) .Where(m => m.Id == matterId && m.Sequence.Count == 0) .ToList(); if (orphanMatter.Count > 0) { db.Matter.Remove(orphanMatter[0]); db.SaveChanges(); } } throw; } finally { Dispose(true); } })); }
public ActionResult Index(long[] matterIds, short characteristicLinkId, short[] characteristicLinkIds, Feature[] features) { return(CreateTask(() => { matterIds = matterIds.OrderBy(m => m).ToArray(); var matterNames = new string[matterIds.Length]; var remoteIds = new string[matterIds.Length]; var subsequencesCharacteristicsNames = new string[characteristicLinkIds.Length]; var subsequencesCharacteristicsList = new SelectListItem[characteristicLinkIds.Length]; var attributeValuesCache = new AttributeValueCacheManager(); long[] sequenceIds; using (var db = new LibiadaWebEntities()) { DnaSequence[] parentSequences = db.DnaSequence.Include(s => s.Matter) .Where(s => s.Notation == Notation.Nucleotides && matterIds.Contains(s.MatterId)) .OrderBy(s => s.MatterId) .ToArray(); for (int n = 0; n < parentSequences.Length; n++) { matterNames[n] = parentSequences[n].Matter.Name; remoteIds[n] = parentSequences[n].RemoteId; } var geneticSequenceRepository = new GeneticSequenceRepository(db); sequenceIds = geneticSequenceRepository.GetNucleotideSequenceIds(matterIds); } var characteristicTypeLinkRepository = FullCharacteristicRepository.Instance; string sequenceCharacteristicName = characteristicTypeLinkRepository.GetCharacteristicName(characteristicLinkId); for (int k = 0; k < characteristicLinkIds.Length; k++) { subsequencesCharacteristicsNames[k] = characteristicTypeLinkRepository.GetCharacteristicName(characteristicLinkIds[k]); subsequencesCharacteristicsList[k] = new SelectListItem { Value = k.ToString(), Text = subsequencesCharacteristicsNames[k], Selected = false }; } double[] characteristics = SequencesCharacteristicsCalculator.Calculate(sequenceIds, characteristicLinkId); var sequencesData = new SequenceData[matterIds.Length]; for (int i = 0; i < matterIds.Length; i++) { // all subsequence calculations SubsequenceData[] subsequencesData = CalculateSubsequencesCharacteristics( characteristicLinkIds, features, sequenceIds[i]); attributeValuesCache.FillAttributeValues(subsequencesData); sequencesData[i] = new SequenceData(matterIds[i], matterNames[i], remoteIds[i], characteristics[i], subsequencesData); } // sorting organisms by their characteristic sequencesData = sequencesData.OrderBy(r => r.Characteristic).ToArray(); List <AttributeValue> allAttributeValues = attributeValuesCache.AllAttributeValues; var resultData = new Dictionary <string, object> { { "result", sequencesData }, { "subsequencesCharacteristicsNames", subsequencesCharacteristicsNames }, { "subsequencesCharacteristicsList", subsequencesCharacteristicsList }, { "sequenceCharacteristicName", sequenceCharacteristicName }, { "features", features.ToSelectList(features).ToDictionary(f => f.Value) }, { "attributes", EnumExtensions.ToArray <Attribute>().ToDictionary(a => (byte)a, a => a.GetDisplayValue()) }, { "attributeValues", allAttributeValues.Select(sa => new { attribute = sa.AttributeId, value = sa.Value }) } }; return new Dictionary <string, object> { { "data", JsonConvert.SerializeObject(resultData) } }; })); }
public ActionResult Index( string searchQuery, bool importPartial, bool filterMinLength, int minLength, bool filterMaxLength, int maxLength) { return(CreateTask(() => { if (filterMinLength) { searchQuery = filterMaxLength ? NcbiHelper.FormatNcbiSearchTerm(searchQuery, minLength, maxLength: maxLength) : NcbiHelper.FormatNcbiSearchTerm(searchQuery, minLength); } else { searchQuery = filterMaxLength ? NcbiHelper.FormatNcbiSearchTerm(searchQuery, maxLength: maxLength) : NcbiHelper.FormatNcbiSearchTerm(searchQuery); } List <NuccoreObject> searchResults = NcbiHelper.ExecuteESummaryRequest(searchQuery, importPartial); List <NuccoreObject> unfilteredSearchResults; List <NuccoreObject> filteresOutSearchResults = searchResults; string[] accessions; if (!importPartial) { unfilteredSearchResults = NcbiHelper.ExecuteESummaryRequest(searchQuery, true); filteresOutSearchResults = unfilteredSearchResults.Except(searchResults).ToList(); accessions = unfilteredSearchResults.Select(no => no.AccessionVersion.Split('.')[0]).Distinct().ToArray(); } else { accessions = searchResults.Select(no => no.AccessionVersion.Split('.')[0]).Distinct().ToArray(); } var results = new List <MatterImportResult>(accessions.Length); string[] existingAccessions; using (var db = new LibiadaWebEntities()) { var dnaSequenceRepository = new GeneticSequenceRepository(db); (existingAccessions, _) = dnaSequenceRepository.SplitAccessionsIntoExistingAndNotImported(accessions); } searchResults = searchResults .Where(sr => !existingAccessions.Contains(sr.AccessionVersion.Split('.')[0])) .ToList(); foreach (var searchResult in searchResults) { results.Add(new MatterImportResult() { MatterName = $"{searchResult.Title} | {searchResult.AccessionVersion}", Result = "Found new sequence", Status = "Success" }); } results.AddRange(existingAccessions.ConvertAll(existingAccession => new MatterImportResult { MatterName = existingAccession, Result = "Sequence already exists", Status = "Exists" })); if (!importPartial) { filteresOutSearchResults = filteresOutSearchResults .Where(sr => !existingAccessions.Contains(sr.AccessionVersion.Split('.')[0])) .ToList(); foreach (var filteresOutSearchResult in filteresOutSearchResults) { results.Add(new MatterImportResult() { MatterName = $"{filteresOutSearchResult.Title} | {filteresOutSearchResult.AccessionVersion}", Result = "Filtered out", Status = "Error" }); } } accessions = searchResults.Select(sr => sr.AccessionVersion).ToArray(); var result = new Dictionary <string, object> { { "result", results }, { "accessions", accessions } }; return new Dictionary <string, string> { { "data", JsonConvert.SerializeObject(result) } }; })); }