public IOutputDescription GetOutputDescription()
{
IOutputDescription result = null;
var dataSourceShape = DataSourceShapeFactory.CreateDataSourceShape();
var requestResponse = Scrubber.Scrub(RequestResponse);
try
{
result = OutputDescriptionFactory.CreateOutputDescription(OutputFormats.ShapedXML);
dataSourceShape = DataSourceShapeFactory.CreateDataSourceShape();
result.DataSourceShapes.Add(dataSourceShape);
var dataBrowser = DataBrowserFactory.CreateDataBrowser();
dataSourceShape.Paths.AddRange(dataBrowser.Map(requestResponse));
}
catch (Exception ex)
{
var dataBrowser = DataBrowserFactory.CreateDataBrowser();
var errorResult = new XElement("Error");
errorResult.Add(ex);
var data = errorResult.ToString();
dataSourceShape.Paths.AddRange(dataBrowser.Map(data));
}
return(result);
}
private void ScrubFile(CliInput cliInput)
{
bool inIsFile = false;
bool outIsFile = false;
TextReader inReader = null;
TextWriter outWriter = null;
try
{
logger.Info("Executing scrub...");
PrepareStreams(ref inReader, ref outWriter, ref inIsFile, ref outIsFile, cliInput);
var scrubber = new Scrubber(inReader, outWriter);
var scrubTimer = new Stopwatch();
scrubTimer.Start();
scrubber.Scrub();
scrubTimer.Stop();
GenerateCompletedOutput(scrubber.Errors.Count, scrubber.Warnings.Count, scrubTimer);
}
catch (SqlParseErrorException ex)
{
LogError(ex);
Console.Error.WriteLine(string.Format("Scrub validation error: {0}", ex.Message));
}
catch (Exception ex)
{
LogError(ex);
Console.Error.WriteLine("Scrub failure: An unhandled error occurred during scrubbing. Please check the log file for more details.");
}
CleanupStreams(ref inReader, ref outWriter, ref inIsFile, ref outIsFile);
}
protected override void ExecutionImpl(IEsbChannel esbChannel, IDSFDataObject dataObject, string inputs, string outputs, out ErrorResultTO tmpErrors, int update)
{
tmpErrors = new ErrorResultTO();
var webRequestResult = string.Empty;
try
{
if (Headers == null)
{
tmpErrors.AddError(ErrorResource.HeadersAreNull);
return;
}
if (QueryString == null)
{
tmpErrors.AddError(ErrorResource.QueryIsNull);
return;
}
var(head, query, _) = ConfigureHttp(dataObject, update);
var url = ResourceCatalog.GetResource <WebSource>(Guid.Empty, SourceId);
if (dataObject.IsDebugMode())
{
AddDebugInputItem(new DebugEvalResult(query, "URL", dataObject.Environment, update));
AddDebugInputItem(new DebugEvalResult(url.Address, "Query String", dataObject.Environment, update));
}
webRequestResult = PerformWebRequest(head, query, url);
}
catch (Exception ex)
{
tmpErrors.AddError(ex.Message);
}
finally
{
tmpErrors.MergeErrors(_errorsTo);
ResponseManager = new ResponseManager
{
OutputDescription = OutputDescription,
Outputs = Outputs,
IsObject = IsObject,
ObjectName = ObjectName
};
}
if (IsResponseBase64)
{
ResponseManager.PushResponseIntoEnvironment(webRequestResult, update, dataObject);
return;
}
webRequestResult = Scrubber.Scrub(webRequestResult);
ResponseManager.PushResponseIntoEnvironment(webRequestResult, update, dataObject);
}
protected override void ExecutionImpl(IEsbChannel esbChannel, IDSFDataObject dataObject, string inputs, string outputs, out ErrorResultTO tmpErrors, int update)
{
_dataObject = dataObject;
tmpErrors = new ErrorResultTO();
var webRequestResult = string.Empty;
try
{
var(head, query, postData, conditions) = GetEnvironmentInputVariables(_dataObject.Environment, update);
var source = ResourceCatalog.GetResource <WebSource>(Guid.Empty, SourceId);
var isManualChecked = Convert.ToBoolean(Settings?.FirstOrDefault(s => s.Name == nameof(IsManualChecked))?.Value);
var isFormDataChecked = Convert.ToBoolean(Settings?.FirstOrDefault(s => s.Name == nameof(IsFormDataChecked))?.Value);
var isUrlEncodedChecked = Convert.ToBoolean(Settings?.FirstOrDefault(s => s.Name == nameof(IsUrlEncodedChecked))?.Value);
if (isManualChecked || isFormDataChecked || isUrlEncodedChecked)
{
var webPostOptions = new WebPostOptions
{
Head = head,
Headers = head?.Select(h => h.Name + ":" + h.Value)?.ToArray() ?? new string[0],
Method = WebRequestMethod.Post,
Parameters = conditions,
Query = query,
Source = source,
PostData = postData,
Settings = Settings,
IsManualChecked = isManualChecked,
IsFormDataChecked = isFormDataChecked,
IsUrlEncodedChecked = isUrlEncodedChecked
};
webRequestResult = PerformWebPostRequest(webPostOptions);
}
}
catch (Exception ex)
{
tmpErrors.AddError(ex.Message);
}
finally
{
tmpErrors.MergeErrors(_errorsTo);
var bytes = webRequestResult.Base64StringToByteArray();
var response = bytes.ReadToString();
ResponseManager = new ResponseManager
{
OutputDescription = OutputDescription,
Outputs = Outputs,
IsObject = IsObject,
ObjectName = ObjectName
};
response = Scrubber.Scrub(response);
ResponseManager.PushResponseIntoEnvironment(response, update, dataObject);
}
}
public void Scrubber_Scrub_EmptyString_ExpectEmptyString()
{
//------------------------Setup ----------------------------------------------------------------------
//------------------------Execute---------------------------------------------------------------
var scrub = Scrubber.Scrub(string.Empty);
//------------------------Assert Results---------------------------------------------------
Assert.AreEqual(string.Empty, scrub);
}
protected override object ExecuteService(List <MethodParameter> methodParameters, out ErrorResultTO errors, IOutputFormatter formater = null)
{
Service.Source = Source;
ExecuteWebRequest(Service, out errors);
string result = String.IsNullOrEmpty(Service.JsonPath) || String.IsNullOrEmpty(Service.JsonPathResult)
? Scrubber.Scrub(Service.RequestResponse)
: Scrubber.Scrub(Service.JsonPathResult);
Service.RequestResponse = null;
return(result);
}
public void ScrubberScrubJsonWhereDataIsArrayWithNameExpectSameJson()
{
//------------Setup for test--------------------------
const string JsonData = "{Colours:[{color: \"red\",value: \"#f00\"},{color: \"green\",value: \"#0f0\"},{color: \"blue\",value: \"#00f\"},{color: \"cyan\",value: \"#0ff\"},{color: \"magenta\",value: \"#f0f\"},{color: \"yellow\",value: \"#ff0\"}," +
"{color: \"black\",value: \"#000\"}]}";
//------------Execute Test---------------------------
var scrub = Scrubber.Scrub(JsonData, ScrubType.JSon);
//------------Assert Results-------------------------
Assert.AreEqual(JsonData, scrub);
}
public void ScrubberScrubJsonWithNonArrayDefinitionExpectedReturnsSameJson()
{
//------------------------Setup ----------------------------------------------------------------------
const string JsonData = "{\"created_at\":\"Mon Jul 16 21:09:33 +0000 2012\",\"id\":224974074361806848,\"id_str\":\"224974074361806848\",\"text\":\"It works! Many thanks @NeoCat\",\"source\":\"web\",\"truncated\":false," +
"\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":634794199,\"id_str\":\"634794199\"},\"geo\":null,\"coordinates\":null," +
"\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"favorited\":false,\"retweeted\":false,\"lang\":\"en\"}";
//------------------------Execute---------------------------------------------------------------
var scrub = Scrubber.Scrub(JsonData, ScrubType.JSon);
//------------------------Assert Results---------------------------------------------------
Assert.AreEqual(JsonData, scrub);
}
public void Scrubber_Scrub_IsBase64String_Where_DataHTML_ExpectSameJson()
{
//------------Setup for test--------------------------
const string HtmlData = "<!DOCTYPE html>\r\n<html lang=\"en\">\r\n <head>\r\n <title>Img Align Attribute<\\/title>\r\n <\\/head>\r\n<body>\r\n <p>This is an example. <img src=\"image.png\" alt=\"Image\" align=\"middle\"> More text right here\r\n <img src=\"image.png\" alt=\"Image\" width=\"100\"\\/>\r\n <\\/body>\r\n<\\/html>";
var jsonBytes = HtmlData.Base64StringToByteArray();
var base64String = jsonBytes.ToBase64String();
//------------Execute Test---------------------------
var scrub = Scrubber.Scrub(base64String);
//------------Assert Results-------------------------
Assert.AreEqual(HtmlData, scrub);
}
public void Scrubber_Scrub_IsBase64String_Where_DataIsArray_With_Name_ExpectSameJson()
{
//------------Setup for test--------------------------
const string JsonData = "{Colours:[{color: \"red\",value: \"#f00\"},{color: \"green\",value: \"#0f0\"},{color: \"blue\",value: \"#00f\"},{color: \"cyan\",value: \"#0ff\"},{color: \"magenta\",value: \"#f0f\"},{color: \"yellow\",value: \"#ff0\"}," +
"{color: \"black\",value: \"#000\"}]}";
var jsonBytes = JsonData.Base64StringToByteArray();
var base64String = jsonBytes.ToBase64String();
//------------Execute Test---------------------------
var scrub = Scrubber.Scrub(base64String);
//------------Assert Results-------------------------
Assert.AreEqual(JsonData, scrub);
}
public void SelectEnumerableValuesAsRelatedUsingEnumerablePathFromXmlFromFoPrimitiveType()
{
var xmlDocument = new XmlDocument();
xmlDocument.LoadXml("<boolean xmlns=\"http://schemas.microsoft.com/2003/10/Serialization/\">false</boolean>");
var testData = Scrubber.Scrub(xmlDocument.InnerXml);
var xmlMapper = new XmlMapper();
var paths = xmlMapper.Map(testData).ToList();
var path = paths.FirstOrDefault();
var xmlNavigator = new XmlNavigator(testData);
var dat2 = xmlNavigator.SelectScalar(path);
Assert.AreEqual("false", dat2);
}
void OnCollisionStay(Collision col)
{
if (!wet)
{
return;
}
// we can only find UV coords of collisions with MeshColliders, not other Collider types
if (col.collider is MeshCollider)
{
var scrubbable = col.gameObject.GetComponent <Scrubbable>();
if (scrubbable == null)
{
return;
}
var tex = scrubbable.GetMask();
if (tex != null)
{
Vector3 centroid = Vector3.zero;
foreach (var contact in col.contacts)
{
centroid += contact.point;
}
centroid /= (float)col.contacts.Length;
RaycastHit hit;
if (!col.contacts[0].otherCollider.Raycast(new Ray(centroid + col.contacts[0].normal, -col.contacts[0].normal), out hit, 5.5f))
{
return;
}
if (hit.collider.gameObject != col.gameObject)
{
return;
}
Vector2 uv = hit.textureCoord;
scrubber.Scrub(scrubbable, uv);
}
}
}
public void SelectEnumerableValuesAsRelatedUsingEnumerablePathFromXmlFromFailingTest()
{
var xmlDocument = new XmlDocument();
xmlDocument.LoadXml("<ArrayOfProduct xmlns:i=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"http://schemas.datacontract.org/2004/07/WebApplication1.Controllers\"><Product><Category>Electronic</Category><Id>1</Id><Name>Television</Name><Price>82000</Price></Product><Product><Category>Electronic</Category><Id>2</Id><Name>Refrigerator</Name><Price>23000</Price></Product><Product><Category>Electronic</Category><Id>3</Id><Name>Mobiles</Name><Price>20000</Price></Product><Product><Category>Electronic</Category><Id>4</Id><Name>Laptops</Name><Price>45000</Price></Product><Product><Category>Electronic</Category><Id>5</Id><Name>iPads</Name><Price>67000</Price></Product><Product><Category>Gift Items</Category><Id>6</Id><Name>Toys</Name><Price>15000</Price></Product></ArrayOfProduct>");
var testData = Scrubber.Scrub(xmlDocument.InnerXml);
var xmlMapper = new XmlMapper();
var paths = xmlMapper.Map(testData).ToList();
var path = paths.FirstOrDefault();
var xmlNavigator = new XmlNavigator(testData);
var data2 = xmlNavigator.SelectEnumerablesAsRelated(paths);
const string expected = "Electronic|Electronic|Electronic|Electronic|Electronic|Gift Items";
var actual = string.Join("|", data2[path].Select(s => s.ToString().Trim()));
Assert.AreEqual(expected, actual);
}
public void Scrubber_Scrub_Json_Where_DataIsArray_With_NoName_ExpectNamedArray()
{
//------------Setup for test--------------------------
const string JsonData = "[{\"created_at\":\"Mon Jul 16 21:09:33 +0000 2012\",\"id\":224974074361806848,\"id_str\":\"224974074361806848\",\"text\":\"It works! Many thanks @NeoCat\",\"source\":\"web\",\"truncated\":false," +
"\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":634794199,\"id_str\":\"634794199\"},\"geo\":null,\"coordinates\":null," +
"\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"favorited\":false,\"retweeted\":false,\"lang\":\"en\"},{\"created_at\":\"Mon Jul 16 21:09:33 +0000 2012\",\"id\":224974074361806848,\"id_str\":\"224974074361806848\",\"text\":\"It works! Many thanks @NeoCat\",\"source\":\"web\",\"truncated\":false," +
"\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":634794199,\"id_str\":\"634794199\"},\"geo\":null,\"coordinates\":null," +
"\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"favorited\":false,\"retweeted\":false,\"lang\":\"en\"}]";
const string ExpectedData = "{UnnamedArrayData:[{\"created_at\":\"Mon Jul 16 21:09:33 +0000 2012\",\"id\":224974074361806848,\"id_str\":\"224974074361806848\",\"text\":\"It works! Many thanks @NeoCat\",\"source\":\"web\",\"truncated\":false," +
"\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":634794199,\"id_str\":\"634794199\"},\"geo\":null,\"coordinates\":null," +
"\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"favorited\":false,\"retweeted\":false,\"lang\":\"en\"},{\"created_at\":\"Mon Jul 16 21:09:33 +0000 2012\",\"id\":224974074361806848,\"id_str\":\"224974074361806848\",\"text\":\"It works! Many thanks @NeoCat\",\"source\":\"web\",\"truncated\":false," +
"\"in_reply_to_status_id\":null,\"in_reply_to_status_id_str\":null,\"in_reply_to_user_id\":null,\"in_reply_to_user_id_str\":null,\"in_reply_to_screen_name\":null,\"user\":{\"id\":634794199,\"id_str\":\"634794199\"},\"geo\":null,\"coordinates\":null," +
"\"place\":null,\"contributors\":null,\"retweet_count\":0,\"favorite_count\":0,\"favorited\":false,\"retweeted\":false,\"lang\":\"en\"}]}";
//------------Execute Test---------------------------
var scrub = Scrubber.Scrub(JsonData);
//------------Assert Results-------------------------
Assert.AreEqual(ExpectedData, scrub);
}
protected override void ExecutionImpl(IEsbChannel esbChannel, IDSFDataObject dataObject, string inputs, string outputs, out ErrorResultTO tmpErrors, int update)
{
_dataObject = dataObject;
tmpErrors = new ErrorResultTO();
var webRequestResult = string.Empty;
try
{
var(head, query, postData, conditions) = GetEnvironmentInputVariables(_dataObject.Environment, update);
var source = ResourceCatalog.GetResource <WebSource>(Guid.Empty, SourceId);
if (IsFormDataChecked)
{
webRequestResult = PerformFormDataWebPostRequest(source, WebRequestMethod.Post, query, head, conditions);
}
else if (IsManualChecked)
{
webRequestResult = PerformManualWebPostRequest(head, query, source, postData);
}
}
catch (Exception ex)
{
tmpErrors.AddError(ex.Message);
}
finally
{
tmpErrors.MergeErrors(_errorsTo);
var bytes = webRequestResult.Base64StringToByteArray();
var response = bytes.ReadToString();
ResponseManager = new ResponseManager
{
OutputDescription = OutputDescription,
Outputs = Outputs,
IsObject = IsObject,
ObjectName = ObjectName
};
response = Scrubber.Scrub(response);
ResponseManager.PushResponseIntoEnvironment(response, update, dataObject);
}
}
public static void ExecuteRequest(WebService service, bool throwError, out ErrorResultTO errors, WebExecuteString webExecute)
{
var headers = new List <string>();
if (service.Headers != null)
{
headers.AddRange(service.Headers.Select(nameValue => nameValue.Name + ":" + SetParameters(service.Method.Parameters, nameValue.Value)).ToList());
}
var requestUrl = SetParameters(service.Method.Parameters, service.RequestUrl);
var requestBody = SetParameters(service.Method.Parameters, service.RequestBody);
var webResponse = webExecute?.Invoke(service.Source as WebSource, service.RequestMethod, requestUrl, requestBody, throwError, out errors, headers.ToArray());
service.RequestResponse = Scrubber.Scrub(webResponse);
if (!String.IsNullOrEmpty(service.JsonPath))
{
service.ApplyPath();
}
errors = new ErrorResultTO();
}
protected override void ExecutionImpl(IEsbChannel esbChannel, IDSFDataObject dataObject, string inputs, string outputs, out ErrorResultTO tmpErrors, int update)
{
tmpErrors = new ErrorResultTO();
if (Headers == null)
{
tmpErrors.AddError(ErrorResource.HeadersAreNull);
return;
}
if (QueryString == null)
{
tmpErrors.AddError(ErrorResource.QueryIsNull);
return;
}
var head = Headers.Select(a => new NameValue(ExecutionEnvironment.WarewolfEvalResultToString(dataObject.Environment.Eval(a.Name, update)), ExecutionEnvironment.WarewolfEvalResultToString(dataObject.Environment.Eval(a.Value, update))));
var query = ExecutionEnvironment.WarewolfEvalResultToString(dataObject.Environment.Eval(QueryString, update));
var url = ResourceCatalog.GetResource <WebSource>(Guid.Empty, SourceId);
if (dataObject.IsDebugMode())
{
AddDebugInputItem(new DebugEvalResult(query, "URL", dataObject.Environment, update));
AddDebugInputItem(new DebugEvalResult(url.Address, "Query String", dataObject.Environment, update));
}
var webRequestResult = PerformWebRequest(head, query, url);
ResponseManager = new ResponseManager
{
OutputDescription = OutputDescription,
Outputs = Outputs,
IsObject = IsObject,
ObjectName = ObjectName
};
webRequestResult = Scrubber.Scrub(webRequestResult);
ResponseManager.PushResponseIntoEnvironment(webRequestResult, update, dataObject);
}
public string ToXML(bool scrub = true)
{
var xml = this.SerializeToXml();
return(scrub ? Scrubber.Scrub(xml, ScrubType.Xml) : xml);
}
public static void ExecuteRequest(WebService service, bool throwError, out ErrorResultTO errors, WebExecuteString webExecute)
{
var headers = new List <string>();
var evaluatedHeaders = new List <INameValue>();
if (service.Headers != null)
{
evaluatedHeaders = service.Headers.Select(o => new NameValue(SetParameters(service.Method.Parameters, o.Name), SetParameters(service.Method.Parameters, o.Value)) as INameValue).ToList();
headers.AddRange(ToHeaderStringList(evaluatedHeaders));
}
var requestUrl = SetParameters(service.Method.Parameters, service.RequestUrl);
var requestBody = string.Empty;
var IsClassicRequestBody = !service.IsManualChecked && !service.IsFormDataChecked && !service.IsUrlEncodedChecked;
if (service.IsManualChecked || IsClassicRequestBody)
{
requestBody = SetParameters(service.Method.Parameters, service.RequestBody);
}
var formDataParameters = new List <IFormDataParameters>();
if ((service.IsFormDataChecked || service.IsUrlEncodedChecked) && service.FormDataParameters != null)
{
var headersHelper = new WebRequestHeadersHelper(service.Headers, evaluatedHeaders);
var evaluated = headersHelper.CalculateFormDataContentType();
headers = ToHeaderStringList(evaluated.ToList());
formDataParameters.AddRange(service.FormDataParameters.Select(o =>
{
if (o is TextParameter textParam)
{
textParam.Key = SetParameters(service.Method.Parameters, textParam.Key);
textParam.Value = SetParameters(service.Method.Parameters, textParam.Value);
return(textParam);
}
else if (o is FileParameter fileParam)
{
fileParam.Key = SetParameters(service.Method.Parameters, fileParam.Key);
fileParam.FileName = SetParameters(service.Method.Parameters, fileParam.FileName);
fileParam.FileBase64 = SetParameters(service.Method.Parameters, fileParam.FileBase64);
return(fileParam);
}
return(o);
}).ToList());
}
var webExecuteStringArgs = new WebExecuteStringArgs
{
IsManualChecked = service.IsManualChecked,
IsFormDataChecked = service.IsFormDataChecked,
IsUrlEncodedChecked = service.IsUrlEncodedChecked,
FormDataParameters = service.FormDataParameters,
WebRequestFactory = null
};
var webResponse = webExecute?.Invoke(service.Source as WebSource, service.RequestMethod, requestUrl, requestBody, throwError, out errors, headers.ToArray(), webExecuteStringArgs);
service.RequestResponse = Scrubber.Scrub(webResponse);
if (!String.IsNullOrEmpty(service.JsonPath))
{
service.ApplyPath();
}
errors = new ErrorResultTO();
}