// Get everything that should be stored as a file in the .msapp. private static IEnumerable <FileEntry> GetMsAppFiles(this CanvasDocument app, ErrorContainer errors) { // Loose files foreach (var file in app._unknownFiles.Values) { yield return(file); } yield return(ToFile(FileKind.Themes, app._themes)); var header = app._header.JsonClone(); header.LastSavedDateTimeUTC = app._entropy.GetHeaderLastSaved(); yield return(ToFile(FileKind.Header, header)); var props = app._properties.JsonClone(); if (app._connections != null) { var json = Utilities.JsonSerialize(app._connections); props.LocalConnectionReferences = json; } if (app._dataSourceReferences != null) { var json = Utilities.JsonSerialize(app._dataSourceReferences); // Some formats serialize empty as "", some serialize as "{}" if (app._dataSourceReferences.Count == 0) { if (app._entropy.LocalDatabaseReferencesAsEmpty) { json = ""; } else { json = "{}"; } } props.LocalDatabaseReferences = json; } if (app._libraryReferences != null) { var json = Utilities.JsonSerialize(app._libraryReferences); props.LibraryDependencies = json; } yield return(ToFile(FileKind.Properties, props)); var(publishInfo, logoFile) = app.TransformLogoOnSave(); yield return(logoFile); if (publishInfo != null) { yield return(ToFile(FileKind.PublishInfo, publishInfo)); } // "DataComponent" data sources are not part of DataSource.json, and instead in their own file var dataSources = new DataSourcesJson { DataSources = app.GetDataSources() .SelectMany(x => x.Value) .Where(x => !x.IsDataComponent) .OrderBy(x => app._entropy.GetOrder(x)) .ToArray() }; yield return(ToFile(FileKind.DataSources, dataSources)); var sourceFiles = new List <SourceFile>(); var idRestorer = new UniqueIdRestorer(app._entropy); var maxPublishOrderIndex = app._entropy.PublishOrderIndices.Any() ? app._entropy.PublishOrderIndices.Values.Max() : 0; // Rehydrate sources before yielding any to be written, processing component defs first foreach (var controlData in app._screens.Concat(app._components) .OrderBy(source => (app._editorStateStore.TryGetControlState(source.Value.Name.Identifier, out var control) && (control.IsComponentDefinition ?? false)) ? -1 : 1)) { var sourceFile = IRStateHelpers.CombineIRAndState(controlData.Value, errors, app._editorStateStore, app._templateStore, idRestorer, app._entropy); // Offset the publishOrderIndex based on Entropy.json foreach (var ctrl in sourceFile.Flatten()) { if (app._entropy.PublishOrderIndices.TryGetValue(ctrl.Name, out var index)) { ctrl.PublishOrderIndex = index; } else { ctrl.PublishOrderIndex = ++maxPublishOrderIndex; } } sourceFiles.Add(sourceFile); } CheckUniqueIds(errors, sourceFiles); // Repair order when screens are unchanged if (sourceFiles.Where(file => !ExcludeControlFromScreenOrdering(file)).Count() == app._screenOrder.Count && sourceFiles.Where(file => !ExcludeControlFromScreenOrdering(file)).All(file => app._screenOrder.Contains(file.ControlName))) { double i = 0.0; foreach (var screen in app._screenOrder) { sourceFiles.First(file => file.ControlName == screen).Value.TopParent.Index = i; i += 1; } } else { // Make up an order, it doesn't really matter. double i = 0.0; foreach (var sourceFile in sourceFiles) { if (ExcludeControlFromScreenOrdering(sourceFile)) { continue; } sourceFile.Value.TopParent.Index = i; i += 1; } } RepairComponentInstanceIndex(app._entropy?.ComponentIndexes ?? new Dictionary <string, double>(), sourceFiles); // This ordering is essential, we need to match the order in which Studio writes the files to replicate certain order-dependent behavior. foreach (var sourceFile in sourceFiles.OrderBy(file => file.GetMsAppFilename())) { yield return(sourceFile.ToMsAppFile()); } var componentTemplates = new List <TemplateMetadataJson>(); foreach (var template in app._templateStore.Contents.Where(template => template.Value.IsComponentTemplate ?? false)) { if (((template.Value.CustomProperties?.Any() ?? false) || template.Value.ComponentAllowCustomization.HasValue) && !(template.Value.ComponentManifest?.IsDataComponent ?? false)) { componentTemplates.Add(template.Value.ToTemplateMetadata(app._entropy)); } } app._templates = new TemplatesJson() { ComponentTemplates = componentTemplates.Any() ? componentTemplates.OrderBy(x => app._entropy.GetComponentOrder(x)).ToArray() : null, UsedTemplates = app._templates.UsedTemplates.OrderBy(x => app._entropy.GetOrder(x)).ToArray() }; yield return(ToFile(FileKind.Templates, app._templates)); var componentsMetadata = new List <ComponentsMetadataJson.Entry>(); var dctemplate = new List <TemplateMetadataJson>(); foreach (var componentTemplate in app._templateStore.Contents.Values.Where(state => state.ComponentManifest != null)) { var manifest = componentTemplate.ComponentManifest; componentsMetadata.Add(new ComponentsMetadataJson.Entry { Name = manifest.Name, TemplateName = manifest.TemplateGuid, ExtensionData = manifest.ExtensionData }); if (manifest.IsDataComponent) { var controlId = GetDataComponentDefinition(sourceFiles.Select(source => source.Value), manifest.TemplateGuid, errors).ControlUniqueId; var template = new TemplateMetadataJson { Name = manifest.TemplateGuid, ComponentType = ComponentType.DataComponent, Version = app._entropy.GetTemplateVersion(manifest.TemplateGuid), IsComponentLocked = false, ComponentChangedSinceFileImport = true, ComponentAllowCustomization = true, CustomProperties = componentTemplate.CustomProperties, DataComponentDefinitionKey = manifest.DataComponentDefinitionKey }; // Rehydrate fields. template.DataComponentDefinitionKey.ControlUniqueId = controlId; dctemplate.Add(template); } } if (componentsMetadata.Count > 0) { // If the components file is present, then write out all files. yield return(ToFile(FileKind.ComponentsMetadata, new ComponentsMetadataJson { Components = componentsMetadata .OrderBy(x => app._entropy.GetOrder(x)) .ToArray() })); } if (dctemplate.Count > 0) { yield return(ToFile(FileKind.DataComponentTemplates, new DataComponentTemplatesJson { ComponentTemplates = dctemplate .OrderBy(x => app._entropy.GetOrder(x)) .ToArray() })); } // Rehydrate the DataComponent DataSource file. { IEnumerable <DataComponentSourcesJson.Entry> ds = from item in app.GetDataSources().SelectMany(x => x.Value).Where(x => x.IsDataComponent) select item.DataComponentDetails; var dsArray = ds.ToArray(); // backcompat-nit: if we have any DC, then always emit the DC Sources file, even if empty. // if (dcmetadataList.Count > 0) if (dctemplate.Count > 0 || dsArray.Length > 0) { yield return(ToFile(FileKind.DataComponentSources, new DataComponentSourcesJson { DataSources = dsArray })); } } if (app._appCheckerResultJson != null) { yield return(ToFile(FileKind.AppCheckerResult, app._appCheckerResultJson)); } if (app._resourcesJson != null) { var resources = app._resourcesJson.JsonClone(); foreach (var resource in resources.Resources) { if (resource.ResourceKind == "LocalFile") { var rootPath = string.Empty; if (app._entropy?.LocalResourceRootPaths.TryGetValue(resource.Name, out rootPath) ?? false) { resource.RootPath = rootPath; } else { resource.RootPath = string.Empty; } } } yield return(ToFile(FileKind.Resources, resources)); } foreach (var assetFile in app._assetFiles) { yield return(new FileEntry { Name = FilePath.RootedAt("Assets", assetFile.Value.Name), RawBytes = assetFile.Value.RawBytes }); } }
public static CanvasDocument Load(Stream streamToMsapp, ErrorContainer errors) { if (streamToMsapp == null) { throw new ArgumentNullException(nameof(streamToMsapp)); } // Read raw files. // Apply transforms. var app = new CanvasDocument(); app._checksum = new ChecksumJson(); // default empty. Will get overwritten if the file is present. app._templateStore = new EditorState.TemplateStore(); app._editorStateStore = new EditorState.EditorStateStore(); ComponentsMetadataJson componentsMetadata = null; DataComponentTemplatesJson dctemplate = null; DataComponentSourcesJson dcsources = null; ChecksumMaker checksumMaker = new ChecksumMaker(); // key = screen, value = index var screenOrder = new Dictionary <string, double>(); ZipArchive zipOpen; try { zipOpen = new ZipArchive(streamToMsapp, ZipArchiveMode.Read); } catch (Exception e) { // Catch cases where stream is corrupted, can't be read, or unavailable. errors.MsAppFormatError(e.Message); return(null); } using (var z = zipOpen) { foreach (var entry in z.Entries) { checksumMaker.AddFile(entry.FullName, entry.ToBytes()); var fullName = entry.FullName; var kind = FileEntry.TriageKind(FilePath.FromMsAppPath(fullName)); switch (kind) { default: // Track any unrecognized files so we can save back. app.AddFile(FileEntry.FromZip(entry)); break; case FileKind.Resources: app._resourcesJson = ToObject <ResourcesJson>(entry); foreach (var resource in app._resourcesJson.Resources) { if (resource.ResourceKind == "LocalFile") { app._entropy.LocalResourceRootPaths.Add(resource.Name, resource.RootPath); resource.RootPath = null; } } break; case FileKind.Asset: app.AddAssetFile(FileEntry.FromZip(entry, name: fullName.Substring("Assets\\".Length))); break; case FileKind.Checksum: app._checksum = ToObject <ChecksumJson>(entry); break; case FileKind.OldEntityJSon: errors.FormatNotSupported($"This is using an older v1 msapp format that is not supported."); throw new DocumentException(); case FileKind.DataComponentTemplates: dctemplate = ToObject <DataComponentTemplatesJson>(entry); break; case FileKind.ComponentsMetadata: componentsMetadata = ToObject <ComponentsMetadataJson>(entry); break; case FileKind.DataComponentSources: dcsources = ToObject <DataComponentSourcesJson>(entry); break; case FileKind.Properties: app._properties = ToObject <DocumentPropertiesJson>(entry); break; case FileKind.Themes: app._themes = ToObject <ThemesJson>(entry); break; case FileKind.Header: app._header = ToObject <HeaderJson>(entry); app._entropy.SetHeaderLastSaved(app._header.LastSavedDateTimeUTC); app._header.LastSavedDateTimeUTC = null; break; case FileKind.PublishInfo: app._publishInfo = ToObject <PublishInfoJson>(entry); break; case FileKind.AppCheckerResult: app._appCheckerResultJson = ToObject <AppCheckerResultJson>(entry); break; case FileKind.ComponentSrc: case FileKind.ControlSrc: case FileKind.TestSrc: { var control = ToObject <ControlInfoJson>(entry); var sf = SourceFile.New(control); // Add to screen order, only screens have meaningful indices, components may have collisions if (!ExcludeControlFromScreenOrdering(sf)) { screenOrder.Add(control.TopParent.Name, control.TopParent.Index); } var flattenedControlTree = sf.Flatten(); foreach (var ctrl in flattenedControlTree) { // Add PublishOrderIndex to Entropy so it doesn't affect the editorstate diff. app._entropy.PublishOrderIndices.Add(ctrl.Name, ctrl.PublishOrderIndex); // For component instances, also track their index in Entropy if (ctrl.Index == 0.0 || ctrl.Template.Id == "http://microsoft.com/appmagic/screen") { continue; } app._entropy.ComponentIndexes.Add(ctrl.Name, ctrl.Index); } IRStateHelpers.SplitIRAndState(sf, app._editorStateStore, app._templateStore, app._entropy, out var controlIR); if (kind == FileKind.ComponentSrc) { app._components.Add(sf.ControlName, controlIR); } else { app._screens.Add(sf.ControlName, controlIR); } } break; case FileKind.DataSources: { var dataSources = ToObject <DataSourcesJson>(entry); Utilities.EnsureNoExtraData(dataSources.ExtensionData); int iOrder = 0; foreach (var ds in dataSources.DataSources) { app.AddDataSourceForLoad(ds, iOrder); iOrder++; } } break; case FileKind.Templates: { app._templates = ToObject <TemplatesJson>(entry); int iOrder = 0; foreach (var template in app._templates.UsedTemplates) { app._entropy.Add(template, iOrder); iOrder++; } iOrder = 0; foreach (var template in app._templates.ComponentTemplates ?? Enumerable.Empty <TemplateMetadataJson>()) { app._entropy.AddComponent(template, iOrder); iOrder++; } } break; } } // foreach zip entry foreach (var componentTemplate in app._templates.ComponentTemplates ?? Enumerable.Empty <TemplateMetadataJson>()) { if (!app._templateStore.TryGetTemplate(componentTemplate.Name, out var template)) { continue; } template.TemplateOriginalName = componentTemplate.OriginalName; template.IsComponentLocked = componentTemplate.IsComponentLocked; template.ComponentChangedSinceFileImport = componentTemplate.ComponentChangedSinceFileImport; template.ComponentAllowCustomization = componentTemplate.ComponentAllowCustomization; template.ComponentExtraMetadata = componentTemplate.ExtensionData; if (template.Version != componentTemplate.Version) { app._entropy.SetTemplateVersion(template.Name, componentTemplate.Version); } } app._screenOrder = screenOrder.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key).ToList(); // Checksums? var currentChecksum = checksumMaker.GetChecksum(); // This is debug only. The server checksum is out of date with the client checksum // The main checksum validation that matters is the repack after unpack #if DEBUG if (app._checksum.ServerStampedChecksum != null && app._checksum.ServerStampedChecksum != currentChecksum.wholeChecksum) { // The server checksum doesn't match the actual contents. // likely has been tampered. errors.ChecksumMismatch("Checksum doesn't match on extract."); if (app._checksum.ServerPerFileChecksums != null) { foreach (var file in app._checksum.ServerPerFileChecksums) { if (!currentChecksum.perFileChecksum.TryGetValue(file.Key, out var fileChecksum)) { errors.ChecksumMismatch("Missing file " + file.Key); } if (fileChecksum != file.Value) { errors.ChecksumMismatch($"File {file.Key} checksum does not match on extract"); } } foreach (var file in currentChecksum.perFileChecksum) { if (!app._checksum.ServerPerFileChecksums.ContainsKey(file.Key)) { errors.ChecksumMismatch("Extra file " + file.Key); } } } } #endif app._checksum.ClientStampedChecksum = currentChecksum.wholeChecksum; app._checksum.ClientPerFileChecksums = currentChecksum.perFileChecksum; // Normalize logo filename. app.TranformLogoOnLoad(); if (app._properties.LibraryDependencies != null) { var refs = Utilities.JsonParse <ComponentDependencyInfo[]>(app._properties.LibraryDependencies); app._libraryReferences = refs; app._properties.LibraryDependencies = null; } if (app._properties.LocalConnectionReferences != null) { var cxs = Utilities.JsonParse <IDictionary <String, ConnectionJson> >(app._properties.LocalConnectionReferences); app._connections = cxs; app._properties.LocalConnectionReferences = null; } if (!string.IsNullOrEmpty(app._properties.LocalDatabaseReferences)) { var dsrs = Utilities.JsonParse <IDictionary <String, LocalDatabaseReferenceJson> >(app._properties.LocalDatabaseReferences); app._dataSourceReferences = dsrs; app._properties.LocalDatabaseReferences = null; app._entropy.LocalDatabaseReferencesAsEmpty = false; } else { app._entropy.LocalDatabaseReferencesAsEmpty = true; } if (componentsMetadata?.Components != null) { int order = 0; foreach (var x in componentsMetadata.Components) { var manifest = ComponentManifest.Create(x); if (!app._templateStore.TryGetTemplate(x.TemplateName, out var templateState)) { errors.FormatNotSupported("Component Metadata contains template not present in the app"); throw new DocumentException(); } templateState.ComponentManifest = manifest; app._entropy.Add(x, order); order++; } } // Only for data-compoents. if (dctemplate?.ComponentTemplates != null) { int order = 0; foreach (var x in dctemplate.ComponentTemplates) { if (x.ComponentType == null) { errors.FormatNotSupported($"Data component {x.Name} is using an outdated format"); throw new DocumentException(); } if (!app._templateStore.TryGetTemplate(x.Name, out var templateState)) { errors.FormatNotSupported("Component Metadata contains template not present in the app"); throw new DocumentException(); } ComponentManifest manifest = templateState.ComponentManifest; // Should already exist app._entropy.SetTemplateVersion(x.Name, x.Version); app._entropy.Add(x, order); manifest.Apply(x); order++; } } if (dcsources?.DataSources != null) { // Component Data sources only appear if the data component is actually // used as a data source in this app. foreach (var x in dcsources.DataSources) { if (x.Type != DataComponentSourcesJson.NativeCDSDataSourceInfo) { throw new NotImplementedException(x.Type); } var ds = new DataSourceEntry { Name = x.Name, DataComponentDetails = x, // pass in all details for full-fidelity Type = DataSourceModel.DataComponentType }; app.AddDataSourceForLoad(ds); } } } app.ApplyAfterMsAppLoadTransforms(errors); app.OnLoadComplete(errors); return(app); }