internal void Validate(List <string> errors, EtlType type) { IsAddingTimeSeries = AddTimeSeries.Regex.Matches(_parent.Script).Count > 0; if (IsAddingTimeSeries && type == EtlType.Sql) { errors.Add("Adding time series isn't supported by SQL ETL"); } FillCollectionToLoadTimeSeriesBehaviorFunction(errors, type); }
internal void Validate(List <string> errors, EtlType type) { IsAddingCounters = AddMethodRegex.Matches(_parent.Script).Count > 0; if (IsAddingCounters && type == EtlType.Sql) { errors.Add("Adding counters isn't supported by SQL ETL"); } FillCollectionToLoadCounterBehaviorFunction(errors, type); }
public virtual bool Validate(ref List <string> errors, EtlType type) { if (errors == null) { throw new ArgumentNullException(nameof(errors)); } if (string.IsNullOrWhiteSpace(Name)) { errors.Add("Script name cannot be empty"); } if (ApplyToAllDocuments) { if (Collections != null && Collections.Count > 0) { errors.Add($"{nameof(Collections)} cannot be specified when {nameof(ApplyToAllDocuments)} is set. Script name: '{Name}'"); } } else { if (Collections == null || Collections.Count == 0) { errors.Add($"{nameof(Collections)} need be specified or {nameof(ApplyToAllDocuments)} has to be set. Script name: '{Name}'"); } } if (string.IsNullOrWhiteSpace(Script) == false) { if (Legacy_ReplicateToMethodRegex.Matches(Script).Count > 0) { errors.Add($"Found `replicateTo<TableName>()` method in '{Name}' script which is not supported. " + "If you are using the SQL replication script from RavenDB 3.x version then please use `loadTo<TableName>()` instead."); } IsAddingAttachments = AddAttachmentMethodRegex.Matches(Script).Count > 0; IsLoadingAttachments = LoadAttachmentMethodRegex.Matches(Script).Count > 0; IsAddingCounters = AddCounterMethodRegex.Matches(Script).Count > 0; if (IsAddingCounters && type == EtlType.Sql) { errors.Add("Adding counters isn't supported by SQL ETL"); } var counterBehaviors = LoadCountersBehaviorMethodRegex.Matches(Script); if (counterBehaviors.Count > 0) { if (type == EtlType.Sql) { errors.Add("Load counter behavior functions aren't supported by SQL ETL"); } else { CollectionToLoadCounterBehaviorFunction = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); for (int i = 0; i < counterBehaviors.Count; i++) { var counterBehaviorFunction = counterBehaviors[i]; if (counterBehaviorFunction.Groups.Count != 2) { errors.Add( "Invalid load counters behavior function. It is expected to have the following signature: " + "loadCountersOf<CollectionName>Behavior(docId, counterName) and return 'true' if counter should be loaded to a destination"); } var functionSignature = counterBehaviorFunction.Groups[0].Value; var collection = counterBehaviorFunction.Groups[1].Value; var functionName = LoadCountersBehaviorMethodNameRegex.Match(functionSignature); if (Collections.Contains(collection) == false) { var scriptCollections = string.Join(", ", Collections.Select(x => ($"'{x}'"))); errors.Add( $"There is '{functionName}' function defined in '{Name}' script while the processed collections " + $"({scriptCollections}) doesn't include '{collection}'. " + "loadCountersOf<CollectionName>Behavior() function is meant to be defined only for counters of docs from collections that " + "are loaded to the same collection on a destination side"); } CollectionToLoadCounterBehaviorFunction[collection] = functionName.Value; } } } var deleteBehaviors = DeleteDocumentsBehaviorMethodRegex.Matches(Script); if (deleteBehaviors.Count > 0) { if (type == EtlType.Sql) { errors.Add("Delete documents behavior functions aren't supported by SQL ETL"); } else { CollectionToDeleteDocumentsBehaviorFunction = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); for (int i = 0; i < deleteBehaviors.Count; i++) { var deleteBehaviorFunction = deleteBehaviors[i]; if (deleteBehaviorFunction.Groups.Count != 2) { errors.Add( "Invalid delete documents behavior function. It is expected to have the following signature: " + "deleteDocumentsOf<CollectionName>Behavior(docId) and return 'true' if document deletion should be sent to a destination"); } var function = deleteBehaviorFunction.Groups[0].Value; var collection = deleteBehaviorFunction.Groups[1].Value; var functionName = DeleteDocumentsBehaviorMethodNameRegex.Match(function); if (Collections.Contains(collection) == false) { var scriptCollections = string.Join(", ", Collections.Select(x => ($"'{x}'"))); errors.Add( $"There is '{functionName}' function defined in '{Name}' script while the processed collections " + $"({scriptCollections}) doesn't include '{collection}'. " + "deleteDocumentsOf<CollectionName>Behavior() function is meant to be defined only for documents from collections that " + "are loaded to the same collection on a destination side"); } CollectionToDeleteDocumentsBehaviorFunction[collection] = functionName.Value; } } } var genericDeleteBehavior = GenericDeleteDocumentsBehaviorMethodRegex.Matches(Script); if (genericDeleteBehavior.Count > 0) { if (type == EtlType.Sql) { errors.Add("Delete documents behavior functions aren't supported by SQL ETL"); } else { if (genericDeleteBehavior.Count > 1) { errors.Add("Generic delete behavior function can be defined just once in the script"); } else { if (CollectionToDeleteDocumentsBehaviorFunction == null) { CollectionToDeleteDocumentsBehaviorFunction = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); } CollectionToDeleteDocumentsBehaviorFunction[GenericDeleteDocumentsBehaviorFunctionKey] = GenericDeleteDocumentsBehaviorFunctionName; } } } var collections = GetCollectionsFromScript(); if (collections == null || collections.Length == 0) { var actualScript = Script; if (deleteBehaviors.Count > 0) { // let's skip all delete behavior functions to check if we have empty transformation for (int i = 0; i < deleteBehaviors.Count; i++) { actualScript = actualScript.Replace(deleteBehaviors[i].Value, string.Empty); } } if (genericDeleteBehavior.Count == 1) { actualScript = actualScript.Replace(genericDeleteBehavior[0].Value, string.Empty); } if (string.IsNullOrWhiteSpace(actualScript) == false) { string targetName; switch (type) { case EtlType.Raven: targetName = "Collection"; break; case EtlType.Sql: targetName = "Table"; break; default: throw new ArgumentException($"Unknown ETL type: {type}"); } errors.Add($"No `loadTo<{targetName}Name>()` method call found in '{Name}' script"); } else { IsEmptyScript = true; } } } else { IsEmptyScript = true; } return(errors.Count == 0); }
protected UpdateEtlCommand(long taskId, T configuration, EtlType type, string databaseName, string uniqueRequestId) : base(databaseName, uniqueRequestId) { TaskId = taskId; Configuration = configuration; EtlType = type; }
private void FillCollectionToLoadCounterBehaviorFunction(List <string> errors, EtlType type) { var counterBehaviors = LoadBehaviorMethodRegex.Matches(_parent.Script); if (counterBehaviors.Count == 0) { return; } if (type == EtlType.Sql) { errors.Add("Load counter behavior functions aren't supported by SQL ETL"); return; } CollectionToLoadBehaviorFunction = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); foreach (Match counterBehaviorFunction in counterBehaviors) { if (counterBehaviorFunction.Groups.Count != 2) { errors.Add( "Invalid load counters behavior function. It is expected to have the following signature: " + "loadCountersOf<CollectionName>Behavior(docId, counterName) and return 'true' if counter should be loaded to a destination"); } var functionSignature = counterBehaviorFunction.Groups[0].Value; var collection = counterBehaviorFunction.Groups[1].Value; var functionName = LoadBehaviorMethodNameRegex.Match(functionSignature); if (_parent.Collections.Contains(collection) == false) { var scriptCollections = string.Join(", ", _parent.Collections.Select(x => ($"'{x}'"))); errors.Add( $"There is '{functionName}' function defined in '{_parent.Name}' script while the processed collections " + $"({scriptCollections}) doesn't include '{collection}'. " + "loadCountersOf<CollectionName>Behavior() function is meant to be defined only for counters of docs from collections that " + "are loaded to the same collection on a destination side"); } else if (_parent.GetCollectionsFromScript().Contains(collection) == false) { errors.Add($"`{functionName}` function where Defined while there is not load to {collection}. Load behavior function apply only if load to default collection"); } if (CollectionToLoadBehaviorFunction.ContainsKey(collection)) { errors.Add($"There are multiple '{functionName}' functions defined"); } CollectionToLoadBehaviorFunction[collection] = functionName.Value; } }
private void FillCollectionToLoadTimeSeriesBehaviorFunction(List <string> errors, EtlType type) { var timeSeriesBehaviors = LoadTimeSeriesOfCollectionBehavior.Regex.Matches(_parent.Script); if (timeSeriesBehaviors.Count == 0) { return; } if (type == EtlType.Sql) { errors.Add("Load time series behavior functions aren't supported by SQL ETL"); return; } CollectionToLoadBehaviorFunction = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase); foreach (Match timeSeriesBehaviorFunction in timeSeriesBehaviors) { var functionName = timeSeriesBehaviorFunction.Groups["func_name"].Value; var collection = timeSeriesBehaviorFunction.Groups["collection"].Value; var args = timeSeriesBehaviorFunction.Groups["param"].Captures; if (args.Count > LoadTimeSeriesOfCollectionBehavior.ParamsCount) { errors.Add($"'{functionName} function defined with {args.Count}. The signature should be {LoadTimeSeriesOfCollectionBehavior.Signature}"); } if (_parent.Collections.Contains(collection) == false) { var scriptCollections = string.Join(", ", _parent.Collections.Select(x => ($"'{x}'"))); errors.Add( $"There is '{functionName}' function defined in '{_parent.Name}' script while the processed collections " + $"({scriptCollections}) doesn't include '{collection}'. " + $"{LoadTimeSeriesOfCollectionBehavior.Signature} function is meant to be defined only for time series of docs from collections that " + "are loaded to the same collection on a destination side"); } else if (_parent.GetCollectionsFromScript().Contains(collection) == false) { errors.Add($"`{functionName}` function where Defined while there is not load to {collection}. Load behavior function apply only if load to default collection"); } if (CollectionToLoadBehaviorFunction.ContainsKey(collection)) { errors.Add($"There are multiple '{functionName}' functions defined"); } CollectionToLoadBehaviorFunction[collection] = functionName; } }