/// <summary>
/// Create an UpdateFile object with metadata from the XML blob and URLs from the url data array
/// </summary>
/// <param name="xdoc">The XML element that holds file metadata</param>
/// <param name="metadataSource">The metadata source that contains files URL information.</param>
/// <returns></returns>
public static List <UpdateFile> Parse(XDocument xdoc, IMetadataSource metadataSource)
{
var parsedFiles = new List <UpdateFile>();
// Grab all File elements from the XML
var fileElements = xdoc.Descendants(XName.Get("File", "http://schemas.microsoft.com/msus/2002/12/Update"));
foreach (var fileElement in fileElements)
{
// Create a new UpdateFile from the File element
parsedFiles.Add(new UpdateFile(fileElement));
}
// Find URLs for the parsed files. Urls are matched by file digest.
foreach (var file in parsedFiles)
{
foreach (var hash in file.Digests)
{
if (metadataSource.HasFile(hash.DigestBase64))
{
file.Urls.Add(metadataSource.GetFile(hash.DigestBase64));
break;
}
}
}
return(parsedFiles);
}
/// <summary>
/// Initialize a new instance of UpdateServerStartup.
/// </summary>
/// <param name="config">
/// <para>ASP.NET configuration.</para>
/// <list type="bullet">
/// <item><description>Required: string entry "metadata-source" with the path to file containing updates metadata.</description></item>
/// <item><description>Required: string entry "server-config" containing the server configuration in JSON format. A reference server configuration file is provided.</description></item>
/// <item><description>Optional: string entry "content-source" containing the path where update content is stored.</description></item>
/// <item><description>Optional: string entry "content-http-root" with the root URL where update content is serverd from (e.g. http://my-update-server.com). Usually this set to the address/name of the server. Required if "content-source" is specified.</description></item>
/// </list>
/// </param>
public UpdateServerStartup(IConfiguration config)
{
// Get the repository path from the configuration
var metadataSourceFile = config.GetValue <string>("metadata-source");
MetadataSource = CompressedMetadataStore.Open(metadataSourceFile);
if (MetadataSource == null)
{
throw new System.Exception($"Cannot open updates metadata source from path {metadataSourceFile}");
}
UpdateServiceConfiguration = Newtonsoft.Json.JsonConvert.DeserializeObject <Config>(config.GetValue <string>("server-config"));
// A file that contains mapping of update identity to a 32 bit, locally assigned revision ID.
var contentPath = config.GetValue <string>("content-source");
if (!string.IsNullOrEmpty(contentPath))
{
ContentSource = new FileSystemContentStore(contentPath);
if (ContentSource == null)
{
throw new System.Exception($"Cannot open updates content source from path {contentPath}");
}
}
ContentRoot = config.GetValue <string>("content-http-root");
}
public void Setup()
{
var sources = new IMetadataSource[] { new QueryingMetadataSource() };
var processors = new IMetadataProcessor[] {};
_provider = new MetadataProvider(sources, processors);
}
public static void PrintFilter(QueryFilter filter, IMetadataSource metadataSource)
{
Console.WriteLine("Filter:");
Console.WriteLine(" Anchor: {0}", filter.Anchor);
ConsoleOutput.WriteGreen(" Classifications:");
if (metadataSource.ClassificationsIndex.Count == filter.ClassificationsFilter.Count)
{
ConsoleOutput.WriteGreen(" all");
}
else
{
foreach (var classificationId in filter.ClassificationsFilter)
{
ConsoleOutput.WriteGreen($" {classificationId}");
ConsoleOutput.WriteGreen($" {metadataSource.ClassificationsIndex.Values.First(c => c.Identity == classificationId).Title}");
}
}
ConsoleOutput.WriteCyan(" Products:");
if (metadataSource.ProductsIndex.Count == filter.ProductsFilter.Count)
{
Console.WriteLine(" all");
}
else
{
foreach (var productId in filter.ProductsFilter)
{
ConsoleOutput.WriteCyan($" {productId}");
ConsoleOutput.WriteCyan($" {metadataSource.ProductsIndex.Values.First(c => c.Identity == productId).Title}");
}
}
}
public void Setup()
{
var sources = new IMetadataSource[] { new QueryingMetadataSource() };
var processors = new IMetadataProcessor[] {};
_provider = new MetadataProvider(sources, processors);
}
/// <summary>
/// Initialize a new instance of UpstreamServerStartup.
/// </summary>
/// <param name="config">
/// <para>ASP.NET configuration.</para>
///
/// <para>Must contain a string entry "metadata-path" with the path to the metadata source to use</para>
///
/// <para>Must contain a string entry "updates-filter" with a JSON serialized updates metadata filter</para>
///
/// <para>Must contain a string entry "service-config-path" with the path to the service configuration JSON</para>
///
/// <para>Can contain a string entry "content-path" with the path to the content store to use if serving update content</para>
/// </param>
public UpstreamServerStartup(IConfiguration config)
{
// Get the metadata source path from the configuration
var sourcePath = config.GetValue <string>("metadata-path");
var contentPath = config.GetValue <string>("content-path");
// Get the filteres to apply to updates; restricts which updates are shared with downstream servers
Filter = MetadataFilter.FromJson(config.GetValue <string>("updates-filter"));
// Open the updates metadata source. It must exist
LocalMetadataSource = CompressedMetadataStore.Open(sourcePath);
if (LocalMetadataSource == null)
{
throw new System.Exception($"Cannot open the specified metadata source at {sourcePath}");
}
var serviceConfigPath = config.GetValue <string>("service-config-path");
ServiceConfiguration = JsonConvert.DeserializeObject <ServerSyncConfigData>(
File.OpenText(serviceConfigPath).ReadToEnd());
if (!string.IsNullOrEmpty(contentPath))
{
LocalContentSource = new FileSystemContentStore(contentPath);
ServiceConfiguration.CatalogOnlySync = false;
}
else
{
ServiceConfiguration.CatalogOnlySync = true;
}
}
internal ContentController(IMetadataSource metadataSource, IUpdateContentSource contentSource, MetadataFilter filter)
{
MetadataSource = metadataSource;
ContentSource = contentSource;
var updatesWithFiles = MetadataSource.GetUpdates(filter).Where(u => u.HasFiles);
UpdateFiles = updatesWithFiles.SelectMany(u => u.Files).Distinct().ToDictionary(
f => $"{f.GetContentDirectoryName().ToLower()}/{f.Digests[0].HexString.ToLower() + System.IO.Path.GetExtension(f.FileName).ToLower()}");
}
public static IUnityContainer ConfigureMetadata(this IUnityContainer container)
{
var metadataSources = new IMetadataSource[]
{
new QueryingMetadataSource()
};
var metadataProcessors = new IMetadataProcessor[] { };
return container.RegisterType<IMetadataProvider, MetadataProvider>(Lifetime.Singleton, new InjectionConstructor(metadataSources, metadataProcessors));
}
public static IUnityContainer ConfigureMetadata(this IUnityContainer container)
{
var metadataSources = new IMetadataSource[]
{
new QueryingMetadataSource()
};
var metadataProcessors = new IMetadataProcessor[] { };
return(container.RegisterType <IMetadataProvider, MetadataProvider>(Lifetime.Singleton, new InjectionConstructor(metadataSources, metadataProcessors)));
}
/// <summary>
/// Instantiate the server and serve updates from the local repo
/// </summary>
/// <param name="metadataSource">The update metadata source to serve updates from</param>
/// <param name="filter">The filter for which updates to serve.</param>
/// <param name="serviceConfig">Service configuration.</param>
public ServerSyncWebService(IMetadataSource metadataSource, MetadataFilter filter, ServerSyncConfigData serviceConfig)
{
MetadataSource = metadataSource;
ServiceConfiguration = serviceConfig;
UpdatesFilter = filter;
Categories = metadataSource.GetCategories();
FilteredUpdates = metadataSource.GetUpdates(filter).ToDictionary(u => u.Identity);
// If an update contains bundled updates, those bundled updates must also be made available to downstream servers
var bundledUpdates = FilteredUpdates.Values.Where(u => u.IsBundle).SelectMany(u => u.BundledUpdates).Distinct().ToList();
foreach (var bundledUpdateId in bundledUpdates)
{
if (!FilteredUpdates.ContainsKey(bundledUpdateId))
{
FilteredUpdates.Add(bundledUpdateId, metadataSource.GetUpdate(bundledUpdateId));
}
}
// Build the lookup tables by products and classifications
ProductsIndex = new Dictionary <Guid, List <Update> >();
ClassificationsIndex = new Dictionary <Guid, List <Update> >();
foreach (var update in FilteredUpdates.Values)
{
if (update.HasProduct)
{
foreach (var productId in update.ProductIds)
{
if (!ProductsIndex.ContainsKey(productId))
{
ProductsIndex[productId] = new List <Update>();
}
ProductsIndex[productId].Add(update);
}
}
if (update.HasClassification)
{
foreach (var classificationId in update.ClassificationIds)
{
if (!ClassificationsIndex.ContainsKey(classificationId))
{
ClassificationsIndex[classificationId] = new List <Update>();
}
ClassificationsIndex[classificationId].Add(update);
}
}
}
}
public void Generate(IMetadataSource source, IMetadataReferenceResolver metadataReferenceResolver, IMetadataWriter writer)
{
if (source == null)
{
throw new ArgumentNullException(nameof(source));
}
if (writer == null)
{
throw new ArgumentNullException(nameof(writer));
}
writer.Write(new IgnoredNamespaceFilter(source, IgnoredNamespaces, metadataReferenceResolver), WriteAssemblyMetadata);
}
private void SetMetaSource(IMetadataSource value)
{
if (value == null)
{
this.uiProcedures.Items.Clear();
this.metaSource = null;
}
else
{
bool hasChanged = (this.metaSource == null) || (this.metaSource.ConnectionString != value.ConnectionString);
this.metaSource = value;
if (hasChanged)
RefreshProcedureList();
}
}
static void PrintBundledUpdates(Update updateWithBundledUpdates, IMetadataSource metadataSource)
{
if (updateWithBundledUpdates.BundledUpdates.Count() > 0)
{
Console.ForegroundColor = ConsoleColor.DarkCyan;
Console.WriteLine(" Bundled updates:");
Console.ResetColor();
foreach (var id in updateWithBundledUpdates.BundledUpdates)
{
Console.WriteLine(" ID : {0}", id);
Console.WriteLine(" Title : {0}", metadataSource.GetUpdateTitle(id));
}
}
}
/// <summary>
/// Create a content controller from the specified metadata and update content sources.
/// </summary>
/// <param name="metadataSource">The source of update metadata. Used to build the list of known files to serve.</param>
/// <param name="contentSource">The source of content. Used to read update content and send it to clients.</param>
public ClientSyncContentController(IMetadataSource metadataSource, IUpdateContentSource contentSource)
{
ContentSource = contentSource;
var updatesWithFiles = metadataSource.GetUpdates().Where(u => u.HasFiles).ToList();
UpdateFiles = updatesWithFiles.SelectMany(u => u.Files).GroupBy(f => f.Digests[0].DigestBase64).Select(g => g.First()).ToDictionary(
f => {
// TODO: fix; hack; this is an internal implementation detail; must be exposed from server-server-sync library
byte[] hashBytes = Convert.FromBase64String(f.Digests[0].DigestBase64);
var cachedContentDirectoryName = string.Format("{0:X}", hashBytes.Last());
return($"{cachedContentDirectoryName.ToLower()}/{f.Digests[0].HexString.ToLower()}");
});
}
/// <summary>
/// Sets the source of update metadata
/// </summary>
/// <param name="metadataSource">The source for updates metadata</param>
public void SetMetadataSource(IMetadataSource metadataSource)
{
MetadataSourceLock.EnterWriteLock();
MetadataSource = metadataSource;
if (MetadataSource != null)
{
// Get leaf updates - updates that have prerequisites and no dependents
LeafUpdatesGuids = MetadataSource.GetLeafUpdates();
// Get non leaft updates: updates that have prerequisites and dependents
NonLeafUpdates = MetadataSource.GetNonLeafUpdates();
// Get root updates: updates that have no prerequisites
RootUpdates = MetadataSource.GetRootUpdates();
// Filter out leaf updates and only retain software ones
var leafSoftwareUpdates = MetadataSource.UpdatesIndex.Values.OfType <SoftwareUpdate>().GroupBy(u => u.Identity.ID).Select(k => k.Key).ToHashSet();
SoftwareLeafUpdateGuids = LeafUpdatesGuids.Where(g => leafSoftwareUpdates.Contains(g)).ToList();
// Get the mapping of update index to identity that is used in the metadata source.
MetadataSourceIndex = MetadataSource.GetIndex();
var latestRevisionSelector = MetadataSourceIndex
.ToDictionary(k => k.Value, v => v.Key)
.GroupBy(p => p.Key.ID)
.Select(group => group.OrderBy(g => g.Key.Revision).Last());
// Create a mapping for index to update GUID
IdToRevisionMap = latestRevisionSelector.ToDictionary(k => k.Key.ID, v => v.Value);
// Create a mapping from GUID to full identity
IdToFullIdentityMap = latestRevisionSelector.ToDictionary(k => k.Key.ID, v => v.Key);
}
else
{
LeafUpdatesGuids = null;
NonLeafUpdates = null;
RootUpdates = null;
SoftwareLeafUpdateGuids = null;
MetadataSourceIndex = null;
IdToRevisionMap = null;
IdToFullIdentityMap = null;
}
MetadataSourceLock.ExitWriteLock();
}
static void PrintBundleChainRecursive(IMetadataSource source, Update update, int recursionIndex)
{
const int indentSize = 4;
if (update.IsBundled)
{
foreach (var parentBundleID in update.BundleParent)
{
Console.CursorLeft = indentSize * recursionIndex + indentSize;
Console.WriteLine("Bundled in : {0}", parentBundleID);
Console.CursorLeft = indentSize * recursionIndex + indentSize;
Console.WriteLine(" : {0}", source.GetUpdateTitle(parentBundleID));
PrintBundleChainRecursive(source, source.GetUpdate(parentBundleID), recursionIndex + 1);
}
}
}
/// <summary>
/// Gets all files for an update, including files in bundled updates (recursive)
/// </summary>
/// <param name="update"></param>
/// <returns></returns>
public static List <UpdateFile> GetAllUpdateFiles(IMetadataSource source, Update update)
{
var filesList = new List <UpdateFile>();
if (update.HasFiles)
{
filesList.AddRange(update.Files);
}
if (update.IsBundle)
{
foreach (var bundledUpdate in update.BundledUpdates)
{
filesList.AddRange(GetAllUpdateFiles(source, source.UpdatesIndex[bundledUpdate]));
}
}
return(filesList);
}
/// <summary>
/// Opens an updates metadata source the path specified on the command line
/// </summary>
/// <param name="sourceOptions">The command line switch that contains the path</param>
/// <returns>A open updates metadata source</returns>
public static IMetadataSource LoadMetadataSourceFromOptions(IMetadataSourceOptions sourceOptions)
{
IMetadataSource source = null;
Console.Write("Opening update metadata source file... ");
try
{
source = CompressedMetadataStore.Open(sourceOptions.MetadataSourcePath);
ConsoleOutput.WriteGreen("Done!");
}
catch (Exception ex)
{
Console.WriteLine();
ConsoleOutput.WriteRed("Cannot open the query result file:");
ConsoleOutput.WriteRed(ex.Message);
}
return(source);
}
internal DriverUpdate(Identity id, IMetadataSource source) : base(id, source)
{
}
public DbProjectGenerator(DbProjectGeneratorProperties properties)
{
this.metadataSource = properties.MetaDataSource;
this.outputFolder = TextUtil.EnsureSuffix(properties.OutputFolder, @"\");
this.selectedTables = properties.SelectedItems;
this.excludedColumnsCriteria = properties.ExcludedColumnsRegex;
this.selectedProcedures = properties.SelectedProcedures ?? new string[0];
this.typeTemplate = properties.TypeTemplate;
this.spAndCatalogTemplate = properties.SpAndCatalogTemplate;
this.appConfigTemplate = properties.AppConfigTemplate;
this.tableItemTemplates = properties.ItemTemplates;
this.selectedViews = new SortedList<string, string>(properties.SelectedViews.Length);
foreach (string view in properties.SelectedViews)
this.selectedViews.Add(view, view);
}
internal Update(Identity id, IMetadataSource source)
{
Identity = id;
MetadataSource = source;
}
private void OnConnected(object sender, MetaDataSourceEventArgs e)
{
this.MetaDataSource = e.MetaDataSource;
if (this.Connected != null)
Connected(this, new EventArgs());
}
public MetaDataSourceEventArgs(IMetadataSource metaDataSource)
{
this.MetaDataSource = metaDataSource;
}
private void OnDisconnected(object sender, MetaDataSourceEventArgs e)
{
this.MetaDataSource = null;
if (this.Disconnected != null)
Disconnected(this, new EventArgs());
}
/// <summary>
/// Create a valid filter for retrieving updates
/// </summary>
/// <param name="options">The user's commandline options with intended filter</param>
/// <param name="metadataSource">Metadata source that contains the list of known categories and classifications</param>
/// <returns>A query filter that can be used to selectively retrieve updates from the upstream server</returns>
private static QueryFilter CreateValidFilterFromOptions(FetchUpdatesOptions options, IMetadataSource metadataSource)
{
var productFilter = new List <Product>();
var classificationFilter = new List <Classification>();
// If a classification is specified then categories is also required, regardless of user option. Add all categories in this case.
bool allProductsRequired = options.ProductsFilter.Count() == 0 || options.ProductsFilter.Contains("all");
// If category is specified then classification is also required, regardless of user option. Add all classifications in this case.
bool allClassificationsRequired = options.ClassificationsFilter.Count() == 0 || options.ClassificationsFilter.Contains("all");
if (allProductsRequired)
{
productFilter.AddRange(metadataSource.ProductsIndex.Values);
}
else
{
foreach (var categoryGuidString in options.ProductsFilter)
{
var categoryGuid = new Guid(categoryGuidString);
var matchingProduct = metadataSource.ProductsIndex.Values.Where(category => category.Identity.ID == categoryGuid);
if (matchingProduct.Count() == 0)
{
throw new Exception($"Could not find a match for product filter {categoryGuidString}");
}
productFilter.Add(matchingProduct.First());
}
}
if (allClassificationsRequired)
{
classificationFilter.AddRange(metadataSource.ClassificationsIndex.Values);
}
else
{
foreach (var classificationGuidString in options.ClassificationsFilter)
{
var classificationGuid = new Guid(classificationGuidString);
var matchingClassification = metadataSource.ClassificationsIndex.Values.Where(classification => classification.Identity.ID == classificationGuid);
if (matchingClassification.Count() == 0)
{
throw new Exception($"Could not find a match for classification filter {classificationGuidString}");
}
classificationFilter.Add(matchingClassification.First());
}
}
return(new QueryFilter(productFilter, classificationFilter));
}
internal Classification(Identity id, IMetadataSource source) : base(id, source)
{
}
public MetadataSourceViewModel(IMetadataSource source)
{
this.source = source;
}
internal SoftwareUpdate(Identity id, IMetadataSource source) : base(id, source)
{
}
internal Product(Identity id, IMetadataSource source) : base(id, source)
{
}
public MayLilyContext(IConfiguration configuration, IMetadataSource source)
: base(configuration, source)
{
}
/// <summary>
/// Given a list of updates to export, it finds all updates bundled with updates to be exported and adds them
/// to the list as well. This is done recursively, until all bundled updates have been included
/// </summary>
/// <param name="updatesToExport">The updates to export. Bundled updates are added to this list</param>
/// <param name="source">The update metadata to export from.</param>
public static void CompleteTheListOfExportUpdates(List <Update> updatesToExport, IMetadataSource source)
{
bool additionalUpdatesFound = false;
do
{
var additionalUpdates = new List <Identity>();
foreach (var selectedUpdate in updatesToExport)
{
if (selectedUpdate.IsBundle)
{
foreach (var bundledUpdate in selectedUpdate.BundledUpdates)
{
if (!updatesToExport.Any(u => u.Identity.Equals(bundledUpdate)))
{
additionalUpdates.Add(bundledUpdate);
}
}
}
}
foreach (var additionalUpdate in additionalUpdates)
{
// Bundled updates should appear in the list before the updates that bundle them
updatesToExport.Insert(0, source.GetUpdate(additionalUpdate));
}
additionalUpdatesFound = additionalUpdates.Count > 0;
} while (additionalUpdatesFound);
}
public WsusExport(IMetadataSource source, ServerSyncConfigData serviceConfiguration)
{
MetadataSource = source;
ServiceConfiguration = serviceConfiguration;
}
internal Detectoid(Identity id, IMetadataSource source) : base(id, source)
{
}
public static IReadOnlyList <IMetadataNamespace> CalculateNonignoredTransitiveClosure(IMetadataSource source, IEnumerable <string> ignoredNamespaces, IMetadataReferenceResolver metadataReferenceResolver)
{
var regexBuilder = (StringBuilder)null;
foreach (var ignoredNamespace in ignoredNamespaces)
{
if (regexBuilder == null)
{
regexBuilder = new StringBuilder(@"\A(?:");
}
else
{
regexBuilder.Append('|');
}
regexBuilder.Append(Regex.Escape(ignoredNamespace));
}
if (regexBuilder == null)
{
return(source.Namespaces);
}
var namespaceIgnoreRegex = new Regex(regexBuilder.Append(@")(\Z|\.)").ToString(), RegexOptions.IgnoreCase);
var ignoredNamespaceLookup = new Dictionary <string, PartiallyIgnoredNamespaceBuilder>();
var includedNamespaces = new List <IMetadataNamespace>(source.Namespaces.Count);
foreach (var ns in source.Namespaces)
{
if (namespaceIgnoreRegex.IsMatch(ns.Name))
{
ignoredNamespaceLookup.Add(ns.Name, new PartiallyIgnoredNamespaceBuilder(ns));
}
else
{
includedNamespaces.Add(ns);
}
}
if (ignoredNamespaceLookup.Count != 0)
{
var visitor = new ReferencedIgnoredMetadataVisitor(ignoredNamespaceLookup, metadataReferenceResolver);
foreach (var ns in includedNamespaces)
{
var prefix = string.IsNullOrEmpty(ns.Name) ? null : TextNode.Root(ns.Name) + ".";
foreach (var type in ns.Types)
{
visitor.VisitNonignoredType(prefix + type.Name, type);
}
}
foreach (var ns in visitor.ignoredNamespaces.Values)
{
if (ns.Types.Count != 0)
{
includedNamespaces.Add(ns);
}
}
}
return(includedNamespaces);
}
public IgnoredNamespaceFilter(IMetadataSource source, IEnumerable <string> ignoredNamespaces, IMetadataReferenceResolver metadataReferenceResolver)
{
this.source = source ?? throw new ArgumentNullException(nameof(source));
this.ignoredNamespaces = ignoredNamespaces ?? throw new ArgumentNullException(nameof(ignoredNamespaces));
this.metadataReferenceResolver = metadataReferenceResolver ?? throw new ArgumentNullException(nameof(metadataReferenceResolver));
}
public EntityContext(IConfiguration configuration, IMetadataSource source)
: base(configuration.ConnectionString, configuration.BackendConfiguration, source.GetModel())
{
}
protected static DbModel BuildModel(IMetadataSource source, ITypeProvider typeProvider = null)
{
return(BuildModel(CreateMetadataProvider(source), typeProvider));
}
protected static DbModel BuildModel(IMetadataSource source, ITypeProvider typeProvider = null)
{
return BuildModel(CreateMetadataProvider(source), typeProvider);
}