/// <summary>
/// Initializes a new instance of the <see cref="ModelsRepositoryClientOptions"/> class.
/// </summary>
/// <param name="version">
/// The <see cref="ServiceVersion"/> of the service API used when
/// making requests.
/// </param>
/// <param name="resolutionOption">The dependency processing options.</param>
public ModelsRepositoryClientOptions(
ServiceVersion version = LatestVersion,
DependencyResolutionOption resolutionOption = DependencyResolutionOption.Enabled)
{
DependencyResolution = resolutionOption;
Version = version;
}
public ModelParser GetParser(DependencyResolutionOption resolutionOption = DependencyResolutionOption.Enabled)
{
ResolverClient client = GetResolver(resolutionOption);
ModelParser parser = new ModelParser
{
DtmiResolver = client.ParserDtmiResolver
};
return(parser);
}
public ResolverClient GetResolver(DependencyResolutionOption resolutionOption = DependencyResolutionOption.Enabled)
{
string repository = _repository;
if (Validations.IsRelativePath(repository))
{
repository = Path.GetFullPath(repository);
}
return(new ResolverClient(
repository,
new ResolverClientOptions(resolutionOption)));
}
public FetchResult Fetch(string dtmi, Uri repositoryUri, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope("LocalModelFetcher.Fetch");
scope.Start();
try
{
var work = new Queue <string>();
if (resolutionOption == DependencyResolutionOption.TryFromExpanded)
{
work.Enqueue(GetPath(dtmi, repositoryUri, true));
}
work.Enqueue(GetPath(dtmi, repositoryUri, false));
string fnfError = string.Empty;
while (work.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();
string tryContentPath = work.Dequeue();
ModelsRepositoryEventSource.Instance.FetchingModelContent(tryContentPath);
if (File.Exists(tryContentPath))
{
return(new FetchResult
{
Definition = File.ReadAllText(tryContentPath, Encoding.UTF8),
Path = tryContentPath
});
}
ModelsRepositoryEventSource.Instance.ErrorFetchingModelContent(tryContentPath);
fnfError = string.Format(CultureInfo.InvariantCulture, StandardStrings.ErrorFetchingModelContent, tryContentPath);
}
throw new RequestFailedException(
$"{string.Format(CultureInfo.InvariantCulture, StandardStrings.GenericGetModelsError, dtmi)} {fnfError}",
new FileNotFoundException(fnfError));
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
public FetchResult Fetch(
string dtmi, Uri repositoryUri, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope("RemoteModelFetcher.Fetch");
scope.Start();
try
{
Queue <string> work = PrepareWork(dtmi, repositoryUri, resolutionOption == DependencyResolutionOption.TryFromExpanded);
string remoteFetchError = string.Empty;
while (work.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();
string tryContentPath = work.Dequeue();
ModelsRepositoryEventSource.Instance.FetchingModelContent(tryContentPath);
try
{
string content = EvaluatePath(tryContentPath, cancellationToken);
return(new FetchResult
{
Definition = content,
Path = tryContentPath
});
}
catch (Exception)
{
remoteFetchError =
$"{string.Format(CultureInfo.InvariantCulture, StandardStrings.GenericGetModelsError, dtmi)} " +
string.Format(CultureInfo.InvariantCulture, StandardStrings.ErrorFetchingModelContent, tryContentPath);
}
}
throw new RequestFailedException(remoteFetchError);
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
public void ClientOptions()
{
DependencyResolutionOption defaultResolutionOption = DependencyResolutionOption.Enabled;
ResolverClientOptions customOptions =
new ResolverClientOptions(DependencyResolutionOption.TryFromExpanded);
int maxRetries = 10;
customOptions.Retry.MaxRetries = maxRetries;
string repositoryUriString = "https://localhost/myregistry/";
Uri repositoryUri = new Uri(repositoryUriString);
ResolverClient defaultClient = new ResolverClient(repositoryUri);
Assert.AreEqual(defaultResolutionOption, defaultClient.ClientOptions.DependencyResolution);
ResolverClient customClient = new ResolverClient(repositoryUriString, customOptions);
Assert.AreEqual(DependencyResolutionOption.TryFromExpanded, customClient.ClientOptions.DependencyResolution);
Assert.AreEqual(maxRetries, customClient.ClientOptions.Retry.MaxRetries);
}
public IDictionary <string, string> Process(IEnumerable <string> dtmis, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken)
{
return(ProcessAsync(dtmis, false, resolutionOption, cancellationToken).EnsureCompleted());
}
public async Task <FetchResult> FetchAsync(
string dtmi, Uri repositoryUri, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken = default)
{
using DiagnosticScope scope = _clientDiagnostics.CreateScope("RemoteModelFetcher.Fetch");
scope.Start();
try
{
Queue <string> work = PrepareWork(dtmi, repositoryUri, resolutionOption == DependencyResolutionOption.TryFromExpanded);
string remoteFetchError = string.Empty;
RequestFailedException requestFailedExceptionThrown = null;
Exception genericExceptionThrown = null;
while (work.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();
string tryContentPath = work.Dequeue();
ModelsRepositoryEventSource.Instance.FetchingModelContent(tryContentPath);
try
{
string content = await EvaluatePathAsync(tryContentPath, cancellationToken).ConfigureAwait(false);
return(new FetchResult()
{
Definition = content,
Path = tryContentPath
});
}
catch (RequestFailedException ex)
{
requestFailedExceptionThrown = ex;
}
catch (Exception ex)
{
genericExceptionThrown = ex;
}
if (genericExceptionThrown != null || requestFailedExceptionThrown != null)
{
remoteFetchError =
$"{string.Format(CultureInfo.InvariantCulture, StandardStrings.GenericGetModelsError, dtmi)} " +
string.Format(CultureInfo.InvariantCulture, StandardStrings.ErrorFetchingModelContent, tryContentPath);
}
}
if (requestFailedExceptionThrown != null)
{
throw new RequestFailedException(
requestFailedExceptionThrown.Status,
remoteFetchError,
requestFailedExceptionThrown.ErrorCode,
requestFailedExceptionThrown);
}
else
{
throw new RequestFailedException(
(int)HttpStatusCode.BadRequest,
remoteFetchError,
null,
genericExceptionThrown);
}
}
catch (Exception ex)
{
scope.Failed(ex);
throw;
}
}
public Task <FetchResult> FetchAsync(string dtmi, Uri repositoryUri, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken = default)
{
return(Task.FromResult(Fetch(dtmi, repositoryUri, resolutionOption, cancellationToken)));
}
public static async Task <int> Validate(FileInfo modelFile, string repo, DependencyResolutionOption deps, bool strict)
{
Parsing parsing = new Parsing(repo);
try
{
FileExtractResult extractResult = parsing.ExtractModels(modelFile);
List <string> models = extractResult.Models;
if (models.Count == 0)
{
Outputs.WriteError("No models to validate.");
return(ReturnCodes.ValidationError);
}
ModelParser parser;
if (models.Count > 1)
{
// Special case: when validating from an array, only use array contents for resolution.
// Setup vanilla parser with no resolution. We get a better error message when a delegate is assigned.
parser = new ModelParser
{
DtmiResolver = (IReadOnlyCollection <Dtmi> dtmis) =>
{
return(Task.FromResult(Enumerable.Empty <string>()));
}
};
}
else
{
parser = parsing.GetParser(resolutionOption: deps);
}
Outputs.WriteOut($"- Validating models conform to DTDL...");
await parser.ParseAsync(models);
if (strict)
{
if (extractResult.ContentKind == JsonValueKind.Array || models.Count > 1)
{
// Related to file path validation.
Outputs.WriteError("Strict validation requires a single root model object.");
return(ReturnCodes.ValidationError);
}
string id = parsing.GetRootId(models[0]);
Outputs.WriteOut($"- Ensuring DTMIs namespace conformance for model \"{id}\"...");
List <string> invalidSubDtmis = Validations.EnsureSubDtmiNamespace(models[0]);
if (invalidSubDtmis.Count > 0)
{
Outputs.WriteError(
$"The following DTMI's do not start with the root DTMI namespace:{Environment.NewLine}{string.Join($",{Environment.NewLine}", invalidSubDtmis)}");
return(ReturnCodes.ValidationError);
}
// TODO: Evaluate changing how file path validation is invoked.
if (Validations.IsRemoteEndpoint(repo))
{
Outputs.WriteError($"Model file path validation requires a local repository.");
return(ReturnCodes.ValidationError);
}
Outputs.WriteOut($"- Ensuring model file path adheres to DMR path conventions...");
string filePathError = Validations.EnsureValidModelFilePath(modelFile.FullName, models[0], repo);
if (filePathError != null)
{
Outputs.WriteError(
$"File \"{modelFile.FullName}\" does not adhere to DMR path conventions. Expecting \"{filePathError}\".");
return(ReturnCodes.ValidationError);
}
}
}
catch (ResolutionException resolutionEx)
{
Outputs.WriteError(resolutionEx.Message);
return(ReturnCodes.ResolutionError);
}
catch (ResolverException resolverEx)
{
Outputs.WriteError(resolverEx.Message);
return(ReturnCodes.ResolutionError);
}
catch (ParsingException parsingEx)
{
IList <ParsingError> errors = parsingEx.Errors;
string normalizedErrors = string.Empty;
foreach (ParsingError error in errors)
{
normalizedErrors += $"{Environment.NewLine}{error.Message}";
}
Outputs.WriteError(normalizedErrors);
return(ReturnCodes.ValidationError);
}
catch (IOException ioEx)
{
Outputs.WriteError(ioEx.Message);
return(ReturnCodes.InvalidArguments);
}
catch (ArgumentException argEx)
{
Outputs.WriteError(argEx.Message);
return(ReturnCodes.InvalidArguments);
}
catch (System.Text.Json.JsonException jsonEx)
{
Outputs.WriteError($"Parsing json-ld content. Details: {jsonEx.Message}");
return(ReturnCodes.InvalidArguments);
}
return(ReturnCodes.Success);
}
public static async Task <int> Export(string dtmi, FileInfo modelFile, string repo, DependencyResolutionOption deps, string output)
{
//check that we have either model file or dtmi
if (string.IsNullOrWhiteSpace(dtmi) && modelFile == null)
{
string invalidArgMsg = "Please specify a value for --dtmi";
Outputs.WriteError(invalidArgMsg);
return(ReturnCodes.InvalidArguments);
}
Parsing parsing = new Parsing(repo);
try
{
if (string.IsNullOrWhiteSpace(dtmi))
{
dtmi = parsing.GetRootId(modelFile);
if (string.IsNullOrWhiteSpace(dtmi))
{
Outputs.WriteError("Model is missing root @id");
return(ReturnCodes.ValidationError);
}
}
IDictionary <string, string> result = await parsing.GetResolver(resolutionOption : deps).ResolveAsync(dtmi);
List <string> resultList = result.Values.ToList();
string normalizedList = string.Join(',', resultList);
string payload = $"[{normalizedList}]";
using JsonDocument document = JsonDocument.Parse(payload, CommonOptions.DefaultJsonParseOptions);
using MemoryStream stream = new MemoryStream();
await JsonSerializer.SerializeAsync(stream, document.RootElement, CommonOptions.DefaultJsonSerializerOptions);
stream.Position = 0;
using StreamReader streamReader = new StreamReader(stream);
string jsonSerialized = await streamReader.ReadToEndAsync();
Outputs.WriteOut(jsonSerialized);
if (!string.IsNullOrEmpty(output))
{
UTF8Encoding utf8WithoutBom = new UTF8Encoding(false);
await File.WriteAllTextAsync(output, jsonSerialized, utf8WithoutBom);
}
}
catch (ResolverException resolverEx)
{
Outputs.WriteError(resolverEx.Message);
return(ReturnCodes.ResolutionError);
}
catch (System.Text.Json.JsonException jsonEx)
{
Outputs.WriteError($"Parsing json-ld content. Details: {jsonEx.Message}");
return(ReturnCodes.InvalidArguments);
}
return(ReturnCodes.Success);
}
private FetchResult Fetch(string dtmi, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken)
{
return(_modelFetcher.Fetch(dtmi, _repositoryUri, resolutionOption, cancellationToken));
}
public ResolverClientOptions(DependencyResolutionOption resolutionOption)
{
DependencyResolution = resolutionOption;
}
private async Task <IDictionary <string, string> > ProcessAsync(
IEnumerable <string> dtmis, bool async, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken)
{
var processedModels = new Dictionary <string, string>();
Queue <string> toProcessModels = PrepareWork(dtmis);
while (toProcessModels.Count != 0)
{
cancellationToken.ThrowIfCancellationRequested();
string targetDtmi = toProcessModels.Dequeue();
if (processedModels.ContainsKey(targetDtmi))
{
ModelsRepositoryEventSource.Instance.SkippingPreprocessedDtmi(targetDtmi);
continue;
}
ModelsRepositoryEventSource.Instance.ProcessingDtmi(targetDtmi);
FetchResult result = async
? await FetchAsync(targetDtmi, resolutionOption, cancellationToken).ConfigureAwait(false)
: Fetch(targetDtmi, resolutionOption, cancellationToken);
if (result.FromExpanded)
{
Dictionary <string, string> expanded = new ModelQuery(result.Definition).ListToDict();
foreach (KeyValuePair <string, string> kvp in expanded)
{
if (!processedModels.ContainsKey(kvp.Key))
{
processedModels.Add(kvp.Key, kvp.Value);
}
}
continue;
}
ModelMetadata metadata = new ModelQuery(result.Definition).ParseModel();
if (resolutionOption >= DependencyResolutionOption.Enabled)
{
IList <string> dependencies = metadata.Dependencies;
if (dependencies.Count > 0)
{
ModelsRepositoryEventSource.Instance.DiscoveredDependencies(string.Join("\", \"", dependencies));
}
foreach (string dep in dependencies)
{
toProcessModels.Enqueue(dep);
}
}
string parsedDtmi = metadata.Id;
if (!parsedDtmi.Equals(targetDtmi, StringComparison.Ordinal))
{
ModelsRepositoryEventSource.Instance.IncorrectDtmiCasing(targetDtmi, parsedDtmi);
string formatErrorMsg =
$"{string.Format(CultureInfo.InvariantCulture, StandardStrings.GenericGetModelsError, targetDtmi)} " +
string.Format(CultureInfo.InvariantCulture, StandardStrings.IncorrectDtmiCasing, targetDtmi, parsedDtmi);
throw new RequestFailedException(formatErrorMsg, new FormatException(formatErrorMsg));
}
processedModels.Add(targetDtmi, result.Definition);
}
return(processedModels);
}
public static async Task <int> Import(FileInfo modelFile, DirectoryInfo localRepo, DependencyResolutionOption deps, bool strict)
{
if (localRepo == null)
{
localRepo = new DirectoryInfo(Path.GetFullPath("."));
}
Parsing parsing = new Parsing(localRepo.FullName);
try
{
ModelParser parser = parsing.GetParser(resolutionOption: deps);
FileExtractResult extractResult = parsing.ExtractModels(modelFile);
List <string> models = extractResult.Models;
if (models.Count == 0)
{
Outputs.WriteError("No models to import.");
return(ReturnCodes.ValidationError);
}
Outputs.WriteOut($"- Validating models conform to DTDL...");
await parser.ParseAsync(models);
if (strict)
{
foreach (string content in models)
{
string id = parsing.GetRootId(content);
Outputs.WriteOut($"- Ensuring DTMIs namespace conformance for model \"{id}\"...");
List <string> invalidSubDtmis = Validations.EnsureSubDtmiNamespace(content);
if (invalidSubDtmis.Count > 0)
{
Outputs.WriteError(
$"The following DTMI's do not start with the root DTMI namespace:{Environment.NewLine}{string.Join($",{Environment.NewLine}", invalidSubDtmis)}");
return(ReturnCodes.ValidationError);
}
}
}
foreach (string content in models)
{
await ModelImporter.ImportAsync(content, localRepo);
}
}
catch (ResolutionException resolutionEx)
{
Outputs.WriteError(resolutionEx.Message);
return(ReturnCodes.ResolutionError);
}
catch (ResolverException resolverEx)
{
Outputs.WriteError(resolverEx.Message);
return(ReturnCodes.ResolutionError);
}
catch (ParsingException parsingEx)
{
IList <ParsingError> errors = parsingEx.Errors;
string normalizedErrors = string.Empty;
foreach (ParsingError error in errors)
{
normalizedErrors += $"{Environment.NewLine}{error.Message}";
}
Outputs.WriteError(normalizedErrors);
return(ReturnCodes.ValidationError);
}
catch (IOException ioEx)
{
Outputs.WriteError(ioEx.Message);
return(ReturnCodes.InvalidArguments);
}
catch (ArgumentException argEx)
{
Outputs.WriteError(argEx.Message);
return(ReturnCodes.InvalidArguments);
}
catch (System.Text.Json.JsonException jsonEx)
{
Outputs.WriteError($"Parsing json-ld content. Details: {jsonEx.Message}");
return(ReturnCodes.InvalidArguments);
}
return(ReturnCodes.Success);
}
public Task <IDictionary <string, string> > ProcessAsync(IEnumerable <string> dtmis, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken)
{
return(ProcessAsync(dtmis, true, resolutionOption, cancellationToken));
}
public IDictionary <string, string> Process(string dtmi, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken)
{
return(ProcessAsync(new List <string> {
dtmi
}, false, resolutionOption, cancellationToken).EnsureCompleted());
}
public async Task <IDictionary <string, string> > ProcessAsync(string dtmi, DependencyResolutionOption resolutionOption, CancellationToken cancellationToken)
{
return(await ProcessAsync(new List <string> {
dtmi
}, true, resolutionOption, cancellationToken).ConfigureAwait(false));
}
