public Schema(Environment env, IndexMap indexMap, SchemaMap subSchemas)
{
Id = 1UL;
Env = env;
IndexMap = indexMap;
SubSchemas = subSchemas;
}
public string SqlSchemaParsermethod(SchemaMap Schema, string[] SchemaReferenceArray, string[] SchemaValueArray, string SchemaMap)
{
string[] ParseOperation = Schema.operation;
StringBuilder sb = new StringBuilder();
foreach (string unit in ParseOperation)
{
if (Regex.IsMatch(unit, @"^\d+$"))
{
string value = SchemaValueArray[Convert.ToInt32(unit)];
sb.Append(value);
sb.Append(" ");
}
else
{
sb.Append(unit);
sb.Append(" ");
}
}
string ParsedSrting = sb.ToString().Trim();
string Result = string.Empty;
//if (SchemaMap == "select")
//{
KPSqlImplementer objImplementor = new KPSqlImplementer();
Result = objImplementor.SqlImplementerMethod(ParsedSrting);
//}
//else if (SchemaMap == "selectfew")
// {
// KPSqlImplementer objImplementor = new KPSqlImplementer();
// Result = objImplementor.SqlImplementerMethod(ParsedSrting);
// }
return(Result);
}
public Schema(Environment env, IndexMap indexMap)
{
Id = env.NextPropertyMapId();
Env = env;
IndexMap = indexMap;
SubSchemas = new SchemaMap();
}
public void SchemaMap_ReadTestData_CreatedMapping()
{
var expectedMapping = new Dictionary <string, string> {
{ "a", "1" }, { "b", "2" }
};
var path = AssemblyHelpers.AssemblyDirectory(Assembly.GetExecutingAssembly(), "TestData", "SchemaMap.json");
var schemaMap = new SchemaMap(path);
CollectionAssert.AreEqual(expectedMapping, schemaMap.Mapping);
}
public override DataTableStore GetDataTable(DataTableStore dt)
{
//Create a Schema Mapping Helper
DataSchemaMapping mapping = new DataSchemaMapping(SchemaMap, Side);
//Get the Rates from Yahoo
Dictionary <string, double> rates = GetFXRates();
//Calculate the Conversion Rate from other base currency
double conversion_rate = 1 / rates[BaseCurrency];
//Create a Sorted List of Rates
var rateList = new List <string>();
foreach (string k in rates.Keys)
{
rateList.Add(k);
}
rateList.Sort();
//Populate the Rates Table
foreach (string k in rateList)
{
var newRow = dt.NewRow();
foreach (DataSchemaItem item in SchemaMap.GetIncludedColumns())
{
string columnName = mapping.MapColumnToDestination(item);
switch (columnName)
{
case "Currency":
{
newRow[item.ColumnName] = DataSchemaTypeConverter.ConvertTo(k, item.DataType);
break;
}
case "Rate":
{
newRow[item.ColumnName] = DataSchemaTypeConverter.ConvertTo(Math.Round(rates[k] * conversion_rate, 4), item.DataType);
break;
}
default:
{
break;
}
}
}
if (dt.Rows.Add(newRow) == DataTableStore.ABORT)
{
break;
}
}
return(dt);
}
public override DataTableStore GetDataTable(DataTableStore dt)
{
dt.AddIdentifierColumn(typeof(string));
var uriHelper = new MailChimpUriHelper(APIKey);
var mapping = new DataSchemaMapping(SchemaMap, Side);
var schema = MailChimpDataSchema.MailChimpListSchema();
helper.SetAuthorizationHeader(APIKey);
var result = helper.GetRequestAsJson(uriHelper.ListServiceUrl);
if (result["lists"] != null)
{
foreach (var item_row in result["lists"])
{
var newRow = dt.NewRow();
foreach (DataSchemaItem item in SchemaMap.GetIncludedColumns())
{
string columnName = mapping.MapColumnToDestination(item);
var schemaItem = schema[columnName];
if (schemaItem.IsSubValue)
{
foreach (var six in item_row[schemaItem.ObjectName])
{
foreach (var sub_item in item_row[schemaItem.ObjectName].Children <JProperty>())
{
if (sub_item.Name.Equals(schemaItem.FieldName))
{
newRow[item.ColumnName] = DataSchemaTypeConverter.ConvertTo(sub_item.Value, item.DataType);
}
}
}
}
else
{
newRow[item.ColumnName] = DataSchemaTypeConverter.ConvertTo(item_row[schemaItem.FieldName], item.DataType);
}
}
if (dt.Rows.AddWithIdentifier(newRow, DataSchemaTypeConverter.ConvertTo <string>(item_row["id"])) == DataTableStore.ABORT)
{
break;
}
}
}
return(dt);
}
public override DataTableStore GetDataTable(DataTableStore dt, DataTableKeySet keyset)
{
dt.AddIdentifierColumn(typeof(string));
var uriHelper = new MailChimpUriHelper(APIKey);
var mapping = new DataSchemaMapping(SchemaMap, Side);
var schema = MailChimpDataSchema.MailChimpMemberSchema();
var hash_helper = new HashHelper(HashHelper.HashType.MD5);
var included_columns = SchemaMap.GetIncludedColumns();
helper.SetAuthorizationHeader(APIKey);
var target_index = mapping.MapColumnToDestination(keyset.KeyColumn);
foreach (var key in keyset.KeyValues)
{
var index = key;
if (target_index.Equals("email_address"))
{
index = hash_helper.GetHashAsString(DataSchemaTypeConverter.ConvertTo <string>(key)).ToLower();
}
try
{
var result = helper.GetRequestAsJson($"{uriHelper.ListServiceUrl}/{ListId}/members/{index}");
var newRow = dt.NewRow();
var id = ProcessRow(mapping, schema, included_columns, result, newRow);
if (dt.Rows.AddWithIdentifier(newRow, id) == DataTableStore.ABORT)
{
break;
}
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.NotFound)
{
continue;
}
throw;
}
}
return(dt);
}
/// <summary>
/// Convert Row to a hierarchical term that corresponds to the row's schema.
/// The Erlang term is in the form:
/// <code>{SchemaName::atom(), [{FieldName::atom(), Value}]}.</code>
/// </summary>
/// <param name="row">Row to convert to Erlang object</param>
/// <param name="targetName">Target to use to lookup field attributes in the row</param>
/// <param name="schemaName">Alternative schema name to use for the outermost name of the schema in the output.</param>
/// <param name="outputDefVals">When false no field values are included in output if they are equal to default values</param>
public static IErlObject ToErlObject(this Row row, string targetName = null, string schemaName = null, bool outputDefVals = true)
{
var list = new ErlList();
for (int i = 0; i < row.Count(); ++i)
{
var fld = row.Schema[i];
var atr = fld[targetName];
var clrVal = row[i];
IErlObject erlVal;
// Primitive types are converted using the schema type mapping functions
if (clrVal is Row)
{
// If the row is nested, traverse the hierarchy recursively:
var child = (Row)clrVal;
if (ts_VisitedRows == null)
{
ts_VisitedRows = new HashSet <Row>(ReferenceEqualityComparer <Row> .Instance);
}
if (ts_VisitedRows.Contains(child))
{
throw new ErlDataAccessException(StringConsts.ERL_CANNOT_CONVERT_TYPES_CYCLE_ERROR
.Args(row.Schema.Name, fld.Name));
}
ts_VisitedRows.Add(child);
try { erlVal = child.ToErlObject(targetName, null, outputDefVals); }
finally { ts_VisitedRows.Remove(child); }
list.Add(erlVal);
}
// Skip default values if instructed to do so with outputDefVals
else if (clrVal != null && (outputDefVals || atr.Default == null || !clrVal.Equals(atr.Default)))
{
erlVal = SchemaMap.ClrToErlValue(atr.BackendType, clrVal);
list.Add(new ErlTuple(new ErlAtom(fld.Name), erlVal));
}
else if (clrVal == null && atr.Default != null)
{
erlVal = SchemaMap.ClrToErlValue(atr.BackendType, atr.Default);
list.Add(new ErlTuple(new ErlAtom(fld.Name), erlVal));
}
}
return(new ErlTuple(new ErlAtom(schemaName ?? row.Schema.Name), list));
}
public override DataTableStore GetDataTable(DataTableStore dt)
{
// Key this Data Set by the Filename
dt.AddIdentifierColumn(typeof(string));
var mapping = new DataSchemaMapping(SchemaMap, Side);
var columns = SchemaMap.GetIncludedColumns();
var modifiedSince = string.IsNullOrEmpty(ModifiedSinceUTC) ? default(DateTime?) : DateTime.SpecifyKind(DateTime.Parse(ModifiedSinceUTC), DateTimeKind.Utc);
using (Session session = GetSession())
{
// Get the Files from the Server
GetFiles(session, session.ListDirectory(Path), RecuirseFolders, dt, mapping, columns, modifiedSince);
}
return(dt);
}
public override DataTableStore GetDataTable(DataTableStore dt)
{
dt.AddIdentifierColumn(typeof(string));
var uriHelper = new MailChimpUriHelper(APIKey);
var mapping = new DataSchemaMapping(SchemaMap, Side);
var schema = MailChimpDataSchema.MailChimpMemberSchema();
var included_columns = SchemaMap.GetIncludedColumns();
int total_items = 0;
int count = 0;
bool abort = false;
helper.SetAuthorizationHeader(APIKey);
do
{
var result = helper.GetRequestAsJson($"{uriHelper.ListServiceUrl}/{ListId}/members?count={PageSize}&offset={count}");
total_items = result["total_items"].ToObject <int>();
if (result["members"] != null)
{
foreach (var item_row in result["members"])
{
count++;
var newRow = dt.NewRow();
var id = ProcessRow(mapping, schema, included_columns, item_row, newRow);
if (dt.Rows.AddWithIdentifier(newRow, id) == DataTableStore.ABORT)
{
abort = true;
break;
}
}
}
} while (!abort && count < total_items);
return(dt);
}
public override DataTableStore GetDataTable(DataTableStore dt)
{
//Set the Datasource Identifier.
dt.AddIdentifierColumn(typeof(int));
DataSchemaMapping mapping = new DataSchemaMapping(SchemaMap, Side);
IList <DataSchemaItem> columns = SchemaMap.GetIncludedColumns();
var schema = DataSourceInfo.GetPipedriveDataSchema(helper);
int start = 0;
bool continue_load = false;
do
{
var result = helper.GetRequestAsJson($"{DataSourceInfo.PipedriveEndpointUrl}&start={start}&limit={PageSize}");
//Loop around your data adding it to the DataTableStore dt object.
foreach (var item_row in result["data"])
{
if (dt.Rows.AddWithIdentifier(mapping, columns,
(item, columnName) =>
{
var pds = schema[columnName];
return(pds.Parser.ParseValue(item_row[columnName]));
}
, item_row["id"].ToObject <int>()) == DataTableStore.ABORT)
{
break;
}
}
continue_load = result["additional_data"]?["pagination"] != null && result["additional_data"]["pagination"]["more_items_in_collection"].ToObject <bool>();
if (continue_load)
{
start = result["additional_data"]["pagination"]["next_start"].ToObject <int>();
}
} while (continue_load);
return(dt);
}
private SchemaParser(string schemafile, SchemaMap map)
{
_filename = schemafile;
XmlDocument doc = new XmlDocument();
doc.Load(schemafile);
XmlNode e = doc.DocumentElement;
if (!e.Name.EndsWith("schema"))
{
throw new Exception("No schema");
}
_prefix = e.Name.Substring(0, e.Name.Length - 6);
_virtualRoot = new XsdSchema(e);
parse(e, _virtualRoot);
expandImports(map);
expandReferences();
}
/// <summary>
/// Convert an Erlang hierarchical term representing a schema to a Row.
/// </summary>
/// <param name="row">Row to update</param>
/// <param name="data">
/// Data to update row with.
/// The data must be in the form {SchemaName::atom, [{FieldName::atom(), Value}]}.
/// </param>
/// <param name="schema">Alternative schema to use in place of row.Schema</param>
/// <param name="targetName">Name of the target for looking up field attributes</param>
/// <param name="schemaName">Alternative name of the top-most 'SchemaName' atom used in the "data".</param>
/// <param name="knownSchemas">List of known schemas to use when initializing a field a DynamicRow type.</param>
public static void Update(this Row row, IErlObject data, Schema schema = null, string targetName = null,
string schemaName = null, Registry <Schema> knownSchemas = null)
{
if (schema == null)
{
schema = row.Schema;
}
if (schemaName == null)
{
schemaName = schema.Name;
}
if (data == null)
{
data = new ErlTuple(new ErlAtom(schemaName), new ErlList());
}
// Input data must be in the form: {SchemaName::atom(), [{FieldName::atom(), Value}]}
// where Value can be any primitive value or a hierarchical value with another Row type.
if (!checkKeyValueTuple(data) || ((ErlTuple)data)[1].TypeOrder != ErlTypeOrder.ErlList)
{
throw new ErlDataAccessException(
StringConsts.ERL_DS_CRUD_RESP_SCH_MISMATCH_ERROR.Args(data.ToString(), schema.Name));
}
var dataList = ((ErlTuple)data)[1] as ErlList;
// Make sure that the first element of the tuple matches the schema name
if (!((ErlTuple)data)[0].ValueAsString.Equals(schemaName))
{
throw new ErlDataAccessException(
StringConsts.ERL_DS_CRUD_RESP_SCH_MISMATCH_ERROR.Args(data.ToString(), schema.Name));
}
// Contains a set of field names that are present in configuration
var presentFieldNames = new HashSet <string>();
foreach (var item in dataList.Where(checkKeyValueTuple).Cast <ErlTuple>())
{
presentFieldNames.Add(item[0].ValueAsString);
}
ErlList newList = null;
foreach (var fld in schema.Where(fd => typeof(Row).IsAssignableFrom(fd.Type)))
{
if (!presentFieldNames.Contains(fld.Name))
{
if (newList == null)
{
newList = (ErlList)dataList.Clone();
}
// Add: {FieldName::atom(), []}
newList.Add(new ErlTuple(new ErlAtom(fld.Name), new ErlList()));
}
}
// If no new items were added to the list use current list:
if (newList == null)
{
newList = dataList;
}
foreach (var item in newList.Where(checkKeyValueTuple).Cast <ErlTuple>())
{
var name = item[0].ValueAsString;
var value = item[1];
var fdef = schema[name];
var attr = fdef[targetName];
if (!attr.Visible || (attr.Metadata != null && attr.Metadata.Navigate("$readonly|$read-only|$read_only").ValueAsBool()))
{
continue;
}
// If this field is defined in the schema as a Row type, then we need to descend into the
// value's hierarchical structure and treat it as a nested row
if (typeof(Row).IsAssignableFrom(fdef.Type))
{
// Get the row associated
Schema chldSch;
var chldRow = row[fdef.Order] as Row;
// If the row has a field of Row type initialized, use its Schema value.
if (chldRow != null)
{
chldSch = chldRow.Schema;
}
else
{
// Otherwise lookup the schema from the given registry
if (!knownSchemas.TryGetValue(name, out chldSch))
{
throw new ErlDataAccessException(
StringConsts.ERL_DS_SCHEMA_NOT_KNOWN_ERROR.Args(name, data.ToString()));
}
// Construct the field's value as dynmiac row of the looked up schema type
chldRow = new DynamicRow(chldSch);
chldRow.ApplyDefaultFieldValues();
row[fdef.Order] = chldRow;
}
if (value.TypeOrder != ErlTypeOrder.ErlList)
{
throw new ErlDataAccessException(
StringConsts.ERL_DS_SCHEMA_INVALID_VALUE_ERROR.Args(chldSch.Name, value));
}
// Recursively update the field's value from given data by using the field's schema:
chldRow.Update(item, chldSch, targetName, knownSchemas: knownSchemas);
}
else
{
// This is a primitive value type
var clr = SchemaMap.ErlToClrValue(value, schema, fdef, null, data);
row.SetFieldValue(fdef, clr);
}
}
}
public SchemaMapInfo Index()
{
MapPermission.ViewMap.AssertAuthorized();
return(SchemaMap.GetMapInfo());
}
private void expandImports(SchemaMap map)
{
foreach (IXsdNode node in GetAllNodes())
{
XsdImportInclude inc = node as XsdImportInclude;
if (inc != null)
{
string sl = inc.SchemaLocation;
if (map != null)
{
if (!(sl.Contains("/") || sl.Contains("\\")))
{
sl = Path.Combine(Path.GetDirectoryName(_filename), sl);
}
SchemaParser schema = map.Get(sl);
if (schema == null)
{
if (map.OpenAllDontAsk)
{
schema = new SchemaParser(sl, map);
map.Add(sl, schema);
}
else
{
Cursor cursor = Mouse.OverrideCursor;
Mouse.OverrideCursor = null;
TaskDialogResult res = TaskDialog.Show(
new TaskDialogOptions
{
AllowDialogCancellation = true,
Owner = Application.Current.MainWindow,
Title = "Update xsd location",
MainInstruction =
"Open schema " + sl + " ?",
//Content = "Old location:\n" + Data.ValidationData.Xsd + "\nUpdated location:\n" + newXsdLocation,
CommandButtons = new[] { "Yes", "Yes, all", "No" },
MainIcon = VistaTaskDialogIcon.Information,
//ExpandedInfo = "Source: " + source.XPath + "\nTarget: " + target.XPath
});
switch (res.CommandButtonResult)
{
case 0: //yes
schema = new SchemaParser(sl, map);
map.Add(sl, schema);
break;
case 1: //open all
map.OpenAllDontAsk = true;
schema = new SchemaParser(sl, map);
map.Add(sl, schema);
break;
case 2: //no
case null: //cancel
break;
}
Mouse.OverrideCursor = cursor;
}
}
if (schema != null)
{
var xn = schema.GetVirtualRoot();
((XsdNode)node).AddKids(xn);
}
}
}
}
}
protected override void ProcessRecord()
{
PSObject ldifEntry = null;
string attrName = null;
string attrValue = null;
while (!ldifStreamReader.EndOfStream)
{
string line = ldifStreamReader.ReadLine();
while (ldifStreamReader.Peek() == 32)
{
string continuation = ldifStreamReader.ReadLine();
line += continuation.Substring(1);
}
// If the version marker is there, ignore it
if (Regex.IsMatch(line, @"^version.+$", RegexOptions.IgnoreCase))
{
continue;
}
// Ignore comment lines
if (Regex.IsMatch(line, @"^#.*$", RegexOptions.IgnoreCase))
{
continue;
}
// If it is DN, then we are starting a new object
if (dnCheck.IsMatch(line))
{
ldifEntry = new PSObject();
// Give object a type name which can be identified later
ldifEntry.TypeNames.Insert(0, "BoothBilt.Utility.LdifTool.LdifEntry");
string dnValue = dnCheck.Match(line).Groups["dnValue"].ToString();
if (IsBase64String(dnValue))
{
dnValue = Base64Decode(dnValue);
// Identify the entry as having a Base64 DN
// This can then be used -- by Select-Object, for example -- to control processing further down the pipeline
if (ldifEntry.TypeNames.Contains("LdifEntry.Base64"))
{
// we are good;
}
else
{
ldifEntry.TypeNames.Add("LdifEntry.Base64");
}
WriteDebug(dnValue);
}
ldifEntry.Properties.Add(new PSNoteProperty("dn", dnValue));
continue;
}
// If it is an empty line, or a line with only whitespace then we are finished for this object
// Ship it!
if (Regex.IsMatch(line, @"^\s*$", RegexOptions.IgnoreCase))
{
WriteObject(ldifEntry);
// get rid of extra blank lines
while (ldifStreamReader.Peek() == 0x0D)
{
ldifStreamReader.ReadLine();
}
continue;
}
if (attributeMatch.IsMatch(line))
{
attrName = attributeMatch.Match(line).Groups["attrName"].ToString();
attrValue = attributeMatch.Match(line).Groups["attrValue"].ToString();
}
else if (binaryAttributeMatch.IsMatch(line))
{
attrName = binaryAttributeMatch.Match(line).Groups["attrName"].ToString();
attrValue = binaryAttributeMatch.Match(line).Groups["attrValue"].ToString();
if (haveMap)
{
if (SchemaMap.ContainsKey(attrName))
{
if (CanConvert(attrName))
{
if (IsBase64String(attrValue))
{
attrValue = Base64Decode(attrValue);
// Identify the entry as having at least one Base64 attribute that has been converted to UTF8
// This can then be used -- by Select-Object, for example -- to control processing further down the pipeline
if (ldifEntry.TypeNames.Contains("LdifEntry.Base64"))
{
// we are good;
}
else
{
ldifEntry.TypeNames.Add("LdifEntry.Base64");
}
}
}
}
}
else
{
attrName = attrName + "_binary";
}
}
else
{
WriteError(new ErrorRecord(new InvalidDataException(string.Format("Invalid line: {0}", line)), "errid", ErrorCategory.InvalidData, ldifEntry));
continue;
}
// check if this is a changetype attribute, and if it is, make sure that it is an add
// otherwise, throw an error for this entry and read to the end of the entry
if (attrName.ToLowerInvariant() == "changetype")
{
string av = attrValue.ToLowerInvariant();
if (av != "add" && av != "ntdsschemaadd")
{
do
{
line = ldifStreamReader.ReadLine();
if (line == null)
{
break;
}
} while (!Regex.IsMatch(line, @"^\s*$"));
WriteError(new ErrorRecord(new InvalidDataException(string.Format("Invalid changetype: {0}", attrValue)), "errid", ErrorCategory.InvalidArgument, ldifEntry));
continue;
}
}
if (ldifEntry.Properties.Match(attrName).Count > 0)
{
string test = ldifEntry.Properties[attrName].TypeNameOfValue;
if (test == "System.String")
{
ArrayList list = new ArrayList();
list.Add(ldifEntry.Properties[attrName].Value);
list.Add(attrValue);
ldifEntry.Properties[attrName].Value = list;
}
else if (test == "System.Collections.ArrayList")
{
PSPropertyInfo info = ldifEntry.Properties[attrName];
if (info.IsSettable)
{
ArrayList lst = (ArrayList)info.Value;
lst.Add(attrValue);
}
}
}
else
{
ldifEntry.Properties.Add(new PSNoteProperty(attrName, attrValue.ToString()));
}
}
}
