static IDataReader GetReaderForPhysicalInstance(PhysicalInstance physicalInstance) { // HACK get this by looking at the IDataImporter addins. string fileName = GetLocalFileName(physicalInstance); if (string.IsNullOrWhiteSpace(fileName)) { return(null); } string ext = System.IO.Path.GetExtension(fileName).ToLower(); if (ext == ".dta") { return(new StataDataReader(fileName)); } else if (ext == ".sav") { return(new SpssDataReader(fileName)); } else if (ext == ".csv") { // TODO Use a stream and close it. string contents = File.ReadAllText(fileName); return(new CsvReader(new StringReader(contents), true)); } else if (ext == ".rdata" || ext == ".rda") { // TODO return(null); } return(null); }
/// <summary> /// Test ID 022, 023 /// Ability to Assign an Administrative Status /// </summary> /// <param name="dataset"></param> /// <returns></returns> public static void AssignAdministrativeStatus(PhysicalInstance dataset) { // Get the Repository API var api = GetRepositoryApiWindows(); // Set the AdministrativeStatus on a specific version of the survey api.CreateTag(dataset.CompositeId, "TheCurrentAdministrativeState"); }
static CodeList CreateCodeList(PhysicalInstance physicalInstance, DataRelationship dataRelationship, Variable variable, VariableStatistic stats) { using (IDataReader reader = GetReaderForPhysicalInstance(physicalInstance)) { if (reader == null) { return(null); } int columnIdx = GetColumnIndex(physicalInstance, dataRelationship, variable); if (columnIdx == -1) { return(null); } var gatherer = new DataValueFrequencyGatherer(reader); var values = gatherer.CountValueFrequencies(columnIdx); var codeList = new CodeList() { AgencyId = variable.AgencyId }; codeList.ItemName.Copy(variable.ItemName); codeList.Label.Copy(variable.Label); // Clear any existing category statistics. stats.UnfilteredCategoryStatistics.Clear(); foreach (var pair in values.OrderBy(x => x.Key)) { string value = pair.Key; int count = pair.Value; // Create the code and category. var category = new Category() { AgencyId = variable.AgencyId }; var code = new Code() { AgencyId = variable.AgencyId }; code.Value = value; code.Category = category; codeList.Codes.Add(code); // Update the statistics with category frequency. var catStats = new CategoryStatistics(); catStats.CategoryValue = value.ToString(); catStats.Frequency = count; stats.UnfilteredCategoryStatistics.Add(catStats); } return(codeList); } }
static string GetLocalFileName(PhysicalInstance physicalInstance) { if (physicalInstance == null || physicalInstance.FileIdentifications == null || physicalInstance.FileIdentifications.Count == 0) { return(string.Empty); } var localFileID = physicalInstance.FileIdentifications[0]; string fileName = string.Empty; if (localFileID.Uri != null) { fileName = localFileID.Uri.ToString(); } if (fileName == null) { return(null); } string fullFileName = null; if (fileName.StartsWith("file:")) { fileName = fileName.Replace("file:///", string.Empty).Replace("file:", string.Empty); } if (!string.IsNullOrEmpty(fileName)) { if (System.IO.File.Exists(fileName)) { fullFileName = fileName; return(fullFileName); } else { Console.WriteLine("Test"); } } else { Console.WriteLine("test"); } return(null); }
/// <summary> /// Update versions and references to a Variable along this path: /// PhysicalInstance /// DataRelationship /// Variable /// VariableStatistic /// Variable /// </summary> /// <param name="variable"></param> public static void UpdateVersionsAndSave(Variable variable, PhysicalInstance physicalInstance, DataRelationship dataRelationship, VariableStatistic variableStatistic) { var client = RepositoryHelper.GetClient(); var now = DateTime.UtcNow; var itemsToRegister = new List <IVersionable>(); itemsToRegister.Add(variable); itemsToRegister.Add(physicalInstance); itemsToRegister.Add(dataRelationship); // Increase version of the variable. variable.Version++; variable.VersionDate = now; // Increase the version and reference of the PhysicalInstance that // references the DataRelationship and VariableStatistic. physicalInstance.Version++; physicalInstance.VersionDate = now; // Increase the version and reference of the DataRelationship that references the variable. dataRelationship.Version++; dataRelationship.VersionDate = now; // Update the reference to the variable. var variableInDataRelationship = dataRelationship.LogicalRecords .SelectMany(x => x.VariablesInRecord) .Where(x => x.Identifier == variable.Identifier) .FirstOrDefault(); variableInDataRelationship.Version = variable.Version; variableInDataRelationship.VersionDate = now; // Increase the version and reference of hte VariableStatistic that references the Variable. if (variableStatistic != null) { variableStatistic.Version++; variableStatistic.VersionDate = now; variableStatistic.VariableReference.Version = variable.Version; itemsToRegister.Add(variableStatistic); } // Register all the changes. var options = new CommitOptions(); options.NamedOptions.Add("RegisterOrReplace"); client.RegisterItems(itemsToRegister, options); }
private static void CreateHtmlAndPdf(PhysicalInstance dataset) { string resourcePath = Path.Combine("Resources", "icons", "types"); string baseFileName = "codebook"; string addinsPath = Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Algenta", "Colectica", "Addins"); // Set some options for the report. var options = new ReportOptions(); options.IsAccessibilityMode = false; var builder = new ReportBuilder(); // Use the custom codebook format. builder.Builders = new List <IDocumentationBuilder>(); builder.Builders.Add(new SampleCustomCodebookBuilder()); // Create English PDF. MultilingualString.CurrentCulture = "en"; byte[] englishPdfBytes = builder.CreatePdf(dataset, resourcePath, addinsPath, options, null, null); string englishPdfFileName = $"{baseFileName}_en.pdf"; File.WriteAllBytes(englishPdfFileName, englishPdfBytes); // Create English HTML. string englishHtml = builder.CreateHtml(dataset, resourcePath, addinsPath, options, null, null); string englishHtmlFileName = $"{baseFileName}_en.html"; File.WriteAllText(englishHtmlFileName, englishHtml); // Create French PDF. MultilingualString.CurrentCulture = "fr"; byte[] frenchPdfBytes = builder.CreatePdf(dataset, resourcePath, addinsPath, options, null, null); string frenchPdfFileName = $"{baseFileName}_fr.pdf"; File.WriteAllBytes(frenchPdfFileName, frenchPdfBytes); // Create French HTML. string frenchHtml = builder.CreateHtml(dataset, resourcePath, addinsPath, options, null, null); string frenchHtmlFileName = $"{baseFileName}_fr.html"; File.WriteAllText(frenchHtmlFileName, frenchHtml); }
public void Convert(PhysicalInstance pi) { if (pi == null) { throw new ArgumentNullException(nameof(pi)); } if (pi.RecordLayouts.Count == 0) { return; } // Create a single DataRelationship for the PhysicalInstance. // Within this, we will add one LogicalRecord per old RecordLayout. var dataRelationship = new DataRelationship() { AgencyId = pi.AgencyId }; pi.DataRelationships.Add(dataRelationship); // For each record layout, add the variables to a new DataRelationship. foreach (var recordLayout in pi.RecordLayouts) { // Create a LogicalRecord with the same descriptive information // as the RecordLayout we are migrating. var logicalRecord = new LogicalRecord { AgencyId = pi.AgencyId }; dataRelationship.LogicalRecords.Add(logicalRecord); logicalRecord.ItemName.Copy(recordLayout.ItemName); logicalRecord.Label.Copy(recordLayout.Label); logicalRecord.Description.Copy(recordLayout.Description); // For each DataItem in the RecordLayout, add a VariablesInRecord entry // to the LogicalRecord. foreach (var dataItem in recordLayout.DataItems) { logicalRecord.VariablesInRecord.Add(dataItem.Variable); } } // Remove the old RecordLayouts. pi.RecordLayouts.Clear(); }
public static void UpdateVariableProperty(Variable variable, PhysicalInstance physicalInstance, DataRelationship dataRelationship, VariableStatistic variableStatistic, string propertyName, string value) { if (propertyName == "ItemName") { variable.ItemName.Current = value; } else if (propertyName == "Label") { variable.Label.Current = value; } else if (propertyName == "Description") { variable.Description.Current = value; } else if (propertyName == "AnalysisUnit") { variable.AnalysisUnit = value; } else if (propertyName == "ResponseUnit") { variable.ResponseUnit = value; } else if (propertyName == "RepresentationType") { if (value == "Text" && variable.RepresentationType != RepresentationType.Text) { // Clear any existing category statistics. variableStatistic.UnfilteredCategoryStatistics.Clear(); variable.RepresentationType = RepresentationType.Text; } else if (value == "Numeric" && variable.RepresentationType != RepresentationType.Numeric) { // Clear any existing category statistics. variableStatistic.UnfilteredCategoryStatistics.Clear(); variable.RepresentationType = RepresentationType.Numeric; } else if (value == "Code" && variable.RepresentationType != RepresentationType.Code) { variable.RepresentationType = RepresentationType.Code; CreateCodeList(variable, physicalInstance, dataRelationship, variableStatistic); } } }
static void CreateCodeList(Variable variable, PhysicalInstance physicalInstance, DataRelationship dataRelationship, VariableStatistic variableStatistic) { // Create and save a code list with all the unique values. var codeList = CreateCodeList(physicalInstance, dataRelationship, variable, variableStatistic); variable.CodeRepresentation.Codes = codeList; if (codeList != null) { var gatherer = new ItemGathererVisitor(); codeList.Accept(gatherer); var client = RepositoryHelper.GetClient(); var repoOptions = new CommitOptions(); repoOptions.NamedOptions.Add("RegisterOrReplace"); client.RegisterItems(gatherer.FoundItems, repoOptions); } }
static int GetColumnIndex(PhysicalInstance physicalInstance, DataRelationship dataRelationship, Variable variable) { var logicalRecord = dataRelationship.LogicalRecords.FirstOrDefault(); if (logicalRecord == null) { return(-1); } var variableInRecord = logicalRecord.VariablesInRecord .Where(x => x.CompositeId == variable.CompositeId) .FirstOrDefault(); if (variableInRecord == null) { return(-1); } return(logicalRecord.VariablesInRecord.IndexOf(variableInRecord)); }
/// <summary> /// Test ID 024 /// Sample permissions for administrative statuses /// </summary> /// <param name="dataset"></param> public static void AssignPermissions(PhysicalInstance dataset) { // Get the Repository API var api = GetRepositoryApiWindows(); // create a new item permission for the survey var permission = new ItemPermission(); permission.Permission = PermissionType.ExclusiveWrite; permission.RoleName = "*****@*****.**"; //permission.RoleName = "yourdomain\\username"; permission.Identifier = dataset.Identifier; permission.AgencyId = dataset.AgencyId; // apply the permission on the Repository var roles = new RepositorySecurityContext(); roles.IdentifierPermissions.Add(permission); api.AddPermissions(roles); }
/// <summary> /// Test ID 018, 019, 020 /// Document edit/version processing metdata /// </summary> /// <param name="dataset"></param> /// <returns></returns> public static async Task ModifyAndVersionDataset(PhysicalInstance dataset) { // update the dataset description dataset.Description["en"] = "Description of the dataset"; dataset.Description["fr"] = "Description de l'ensemble de données"; // add descriptions to variables. All the same content in this sample var variables = dataset.DataRelationships.SelectMany(x => x.LogicalRecords).SelectMany(l => l.VariablesInRecord); int position = 1; foreach (Variable variable in variables) { variable.Description["en"] = $"This variable is in position {position}."; variable.Description["fr"] = $"Cette variable est en position {position}."; position++; } // The test from 001, 002, and 003 will find the changed items and add them to the Repository // The Repository will record new versions of existing items, and update all the references to the correct version await RegisterItems(dataset); }
public ActionResult Deserialize(SerializeModel model, string command, HttpPostedFileBase postedFile) { if (postedFile != null) { try { string fileExtension = Path.GetExtension(postedFile.FileName); if (fileExtension != ".xml") { return(View(model)); } string row; using (var sreader = new StreamReader(postedFile.InputStream)) { row = sreader.ReadLine(); } model.Xml = row; xml = row; return(View(model)); } catch (Exception ex) { ViewBag.Message = ex.Message; } } switch (command) { case "Process": PhysicalInstance oInfoDTO = new PhysicalInstance(); model.XmlValues = (PhysicalInstance)XMLToObject(xml, oInfoDTO); return(View(model)); default: break; } return(View(model)); }
/// <summary> /// Test ID 025, 026, 027, 028 /// Render PDF and HTML /// </summary> /// <param name="dataset"></param> public static void CreateHtmlAndPdf(PhysicalInstance dataset) { string resourcePath = Path.Combine("Resources", "icons", "types"); string baseFileName = "codebook"; // Set some options for the report. var options = new ReportOptions(); options.IsAccessibilityMode = false; var builder = new ReportBuilder(); // Create English PDF. MultilingualString.CurrentCulture = "en"; byte[] englishPdfBytes = builder.CreatePdf(dataset, resourcePath, null, options, null, null); string englishPdfFileName = $"{baseFileName}_en.pdf"; File.WriteAllBytes(englishPdfFileName, englishPdfBytes); // Create English HTML. string englishHtml = builder.CreateHtml(dataset, resourcePath, null, options, null, null); string englishHtmlFileName = $"{baseFileName}_en.html"; File.WriteAllText(englishHtmlFileName, englishHtml); // Create French PDF. MultilingualString.CurrentCulture = "fr"; byte[] frenchPdfBytes = builder.CreatePdf(dataset, resourcePath, null, options, null, null); string frenchPdfFileName = $"{baseFileName}_fr.pdf"; File.WriteAllBytes(frenchPdfFileName, frenchPdfBytes); // Create French HTML. string frenchHtml = builder.CreateHtml(dataset, resourcePath, null, options, null, null); string frenchHtmlFileName = $"{baseFileName}_fr.html"; File.WriteAllText(frenchHtmlFileName, frenchHtml); }
void GetReferencingItems(RepositoryClientBase client, IdentifierTriple variableId, out PhysicalInstance physicalInstance, out DataRelationship dataRelationship, out VariableStatistic variableStatistic) { dataRelationship = null; variableStatistic = null; physicalInstance = VariableMapper.GetPhysicalInstanceWithVariable(variableId, client); foreach (var dr in physicalInstance.DataRelationships) { client.PopulateItem(dr, false, ChildReferenceProcessing.Populate); if (dr.LogicalRecords .SelectMany(x => x.VariablesInRecord) .Any(x => x.CompositeId == variableId)) { dataRelationship = dr; } } foreach (var stats in physicalInstance.Statistics) { client.PopulateItem(stats); if (stats.AgencyId == variableId.AgencyId & stats.VariableReference.Identifier == variableId.Identifier) { variableStatistic = stats; break; } } }
public void Map(ManagedFile file, PhysicalInstance pi) { if (file == null) { throw new ArgumentNullException("file"); } if (pi == null) { throw new ArgumentNullException("physicalInstance"); } // Map properties from the ManagedFile to DDI. pi.SetUserId("FileNumber", file.Number?.ToString()); pi.DublinCoreMetadata.Title.Current = file.Title; pi.DublinCoreMetadata.AlternateTitle.Current = file.PublicName; var fileId = pi.FileIdentifications.FirstOrDefault(); if (fileId == null) { fileId = new DataFileIdentification(); pi.FileIdentifications.Add(fileId); } Uri uri; bool gotUri = Uri.TryCreate(file.PersistentLink, UriKind.RelativeOrAbsolute, out uri); if (gotUri) { fileId.Uri = uri; } pi.SetUserAttribute("PersistentLinkDate", file.PersistentLinkDate?.ToString()); pi.SetUserAttribute("FileType", file.Type); pi.SetUserAttribute("FormatName", file.FormatName); pi.SetUserAttribute("FormatId", file.FormatId); pi.SetUserAttribute("Size", file.Size.ToString()); pi.SetUserAttribute("CreationDate", file.CreationDate); pi.SetUserAttribute("KindOfData", file.KindOfData); pi.DublinCoreMetadata.Source.Current = file.Source; pi.SetUserAttribute("SourceInformation", file.SourceInformation); pi.DublinCoreMetadata.Rights.Current = file.Rights; fileId.IsPublic = file.IsPublicAccess; pi.SetUserAttribute("UploadedDate", file.UploadedDate); pi.SetUserAttribute("ExternalDatabase", file.ExternalDatabase); pi.SetUserAttribute("Software", file.Software); pi.SetUserAttribute("SoftwareVersion", file.SoftwareVersion); pi.SetUserAttribute("Hardware", file.Hardware); var fingerprint = pi.Fingerprints.FirstOrDefault(); if (fingerprint == null) { fingerprint = new Fingerprint(); pi.Fingerprints.Add(fingerprint); } fingerprint.FingerprintValue = file.Checksum; fingerprint.AlgorithmSpecification = file.ChecksumMethod; pi.SetUserAttribute("ChecksumDate", file.ChecksumDate); pi.SetUserAttribute("VirusCheckOutcome", file.VirusCheckOutcome); pi.SetUserAttribute("VirusCheckMethod", file.VirusCheckMethod); pi.SetUserAttribute("VirusCheckDate", file.VirusCheckDate); pi.SetUserAttribute("AcceptedDate", file.AcceptedDate); pi.SetUserAttribute("CertifiedDate", file.CertifiedDate); }
public ResourcePackage Import(string path, string agencyId) { this.harmonizingCache = new HarmonizingCache(MultilingualString.CurrentCulture); var resourcePackage = new ResourcePackage(); resourcePackage.AgencyId = agencyId; logger.Debug("Importing RData"); if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException("fileName"); } if (!File.Exists(path)) { throw new ArgumentException("The specified file must exist"); } string fileNameWithExtension = Path.GetFileName(path); string fileNameOnly = Path.GetFileNameWithoutExtension(path); logger.Debug("RData import file: " + fileNameOnly); resourcePackage.DublinCoreMetadata.Title.Current = fileNameOnly; // Create the PhysicalInstance. var physicalInstance = new PhysicalInstance() { AgencyId = agencyId }; resourcePackage.PhysicalInstances.Add(physicalInstance); physicalInstance.DublinCoreMetadata.Title.Current = fileNameOnly; // File location if (path != null) { DataFileIdentification fileID = new DataFileIdentification(); Uri uri; if (Uri.TryCreate(path, UriKind.RelativeOrAbsolute, out uri)) { fileID.Uri = uri; } fileID.Path = path; physicalInstance.FileIdentifications.Add(fileID); } // Create the DataRelationship. var dataRelationship = new DataRelationship(); physicalInstance.DataRelationships.Add(dataRelationship); dataRelationship.AgencyId = agencyId; dataRelationship.Label.Current = fileNameOnly; // Load the file into R. string pathForR = path.Replace("\\", "/"); engine.Evaluate(string.Format("load('{0}')", pathForR)); // Find all the data frames. var dataFrames = GetDataFrames(); // For each data frame in the RData file, create a LogicalRecord. foreach (var pair in dataFrames) { string name = pair.Key; var dataFrame = pair.Value; // TODO This should be tracked per record, not PhysicalInstance. physicalInstance.FileStructure.CaseQuantity = dataFrame.RowCount; var logicalRecord = new LogicalRecord() { AgencyId = agencyId }; dataRelationship.LogicalRecords.Add(logicalRecord); logicalRecord.Label.Current = name; List <string> variableLabels = null; var variableLabelsExpr = dataFrame.GetAttribute("var.labels"); if (variableLabelsExpr != null) { var labelVector = variableLabelsExpr.AsVector(); variableLabels = new List <string>(labelVector.Select(x => (string)x)); } for (int i = 0; i < dataFrame.ColumnCount; i++) { string columnName = dataFrame.ColumnNames[i]; var column = dataFrame[i]; var variable = new Variable() { AgencyId = agencyId }; logicalRecord.VariablesInRecord.Add(variable); // Name variable.ItemName.Current = columnName; // Label if (variableLabels != null) { variable.Label.Current = variableLabels[i]; } // Type if (column.Type == RDotNet.Internals.SymbolicExpressionType.NumericVector) { variable.RepresentationType = RepresentationType.Numeric; variable.Additivity = AdditivityType.Stock; } else if (column.Type == RDotNet.Internals.SymbolicExpressionType.IntegerVector) { if (column.IsFactor()) { variable.RepresentationType = RepresentationType.Code; string[] factors = column.AsFactor().GetLevels(); variable.CodeRepresentation.Codes = GetCodeList(factors, agencyId, resourcePackage); } else { variable.RepresentationType = RepresentationType.Numeric; variable.NumericRepresentation.NumericType = NumericType.Integer; variable.Additivity = AdditivityType.Stock; } } else if (column.Type == RDotNet.Internals.SymbolicExpressionType.CharacterVector) { variable.RepresentationType = RepresentationType.Text; } } } return(resourcePackage); }
public static string CreateOrUpdatePhysicalInstanceForFile <TImporter>(Guid fileId, string filePath, string agencyId, PhysicalInstance existingPhysicalInstance) where TImporter : IDataImporter, new() { var logger = LogManager.GetLogger("Curation"); logger.Debug("File import: " + fileId); IDataImporter importer = new TImporter(); var client = RepositoryHelper.GetClient(); PhysicalInstance physicalInstance = existingPhysicalInstance; // For PhysicalInstances that do not exist yet, create from scratch. if (physicalInstance == null) { // Extract metadata. ResourcePackage rp = importer.Import(filePath, agencyId); logger.Debug("Imported metadata from data file."); if (rp.PhysicalInstances.Count == 0) { logger.Debug("No dataset could be extracted from SPSS"); return(string.Empty); } physicalInstance = rp.PhysicalInstances[0]; physicalInstance.Identifier = fileId; } else { // If there is an existing PhysicalInstance, update it from the data file. var updateCommand = new UpdatePhysicalInstanceFromFile(); updateCommand.DataImporters = new List <IDataImporter>(); updateCommand.DataImporters.Add(new SpssImporter()); updateCommand.DataImporters.Add(new StataImporter()); updateCommand.DataImporters.Add(new SasImporter()); updateCommand.DataImporters.Add(new RDataImporter()); updateCommand.DataImporters.Add(new CsvImporter()); var context = new Algenta.Colectica.ViewModel.Commands.VersionableCommandContext(); // Update the PhysicalInstance from the data file. var repoNode = new RepositoryNode(client, null); var repoItemNode = new RepositoryItemNode(existingPhysicalInstance.GetMetadata(), repoNode, client); context.Node = repoItemNode; context.Item = physicalInstance; try { updateCommand.Execute(context); physicalInstance.Version++; foreach (var item in updateCommand.Result.ModifiedItems) { item.Version++; } } catch (Exception ex) { logger.Error("Problem updating PhysicalInstance.", ex); return("Problem updating PhysicalInstance. " + ex.Message); } } // Calculate summary statistics, for both new and updated files. try { logger.Debug("Calculating summary statistics"); var calculator = new PhysicalInstanceSummaryStatisticComputer(); SummaryStatisticsOptions options = new SummaryStatisticsOptions() { CalculateQuartiles = true }; var stats = calculator.ComputeStatistics(importer, filePath, physicalInstance, physicalInstance.FileStructure.CaseQuantity, options, (percent, message) => { }); logger.Debug("Done calculating summary statistics"); if (stats != null) { physicalInstance.Statistics.Clear(); foreach (VariableStatistic stat in stats) { physicalInstance.Statistics.Add(stat); } } } catch (Exception ex) { logger.Error("Problem calculating summary statistics.", ex); return("Problem calculating summary statistics. " + ex.Message); } // Register all items that were created with the repository. DirtyItemGatherer visitor = new DirtyItemGatherer(false, true); physicalInstance.Accept(visitor); logger.Debug("Setting agency IDs"); // The static default agency id is not thread safe, so set it explicitly here foreach (var item in visitor.DirtyItems) { item.AgencyId = agencyId; } logger.Debug("Done setting agency IDs"); logger.Debug("Registering items with the repository"); var repoOptions = new Algenta.Colectica.Model.Repository.CommitOptions(); repoOptions.NamedOptions.Add("RegisterOrReplace"); client.RegisterItems(visitor.DirtyItems, repoOptions); logger.Debug("Done registering items with the repository"); logger.Debug("Done with CreatePhysicalInstanceForFile"); return(string.Empty); }