internal string[] AddClassesToSchema(string schemaName, ClassCollection classCollection)
{
List <string> notAdded = new List <string>();
FeatureSchema fsc = null;
var fidx = _schemas.IndexOf(schemaName);
if (fidx >= 0)
{
fsc = _schemas[fidx];
}
else
{
fsc = new FeatureSchema(schemaName, "");
AddSchema(fsc);
}
foreach (ClassDefinition cls in classCollection)
{
var cidx = fsc.Classes.IndexOf(cls.Name);
if (cidx >= 0)
{
notAdded.Add(cls.Name);
}
else
{
ClassDefinition copy = FdoSchemaUtil.CloneClass(cls, false);
AddClass(fsc.Name, copy);
}
}
return(notAdded.ToArray());
}
public void TestCloneFeatureClassNoGeometry()
{
FeatureClass fc = new FeatureClass("Test", "Test Feature Class");
DataPropertyDefinition id = new DataPropertyDefinition("ID", "");
DataPropertyDefinition name = new DataPropertyDefinition("Name", "");
id.DataType = DataType.DataType_Int32;
id.IsAutoGenerated = true;
id.ReadOnly = true;
id.Nullable = false;
name.DataType = DataType.DataType_String;
name.Length = 255;
name.Nullable = true;
fc.Properties.Add(id);
fc.Properties.Add(name);
fc.IdentityProperties.Add(id);
ClassDefinition cd = FdoSchemaUtil.CloneClass(fc);
Assert.IsNotNull(cd);
Assert.AreEqual(cd.Name, fc.Name);
Assert.AreEqual(cd.Description, fc.Description);
Assert.AreEqual(cd.Properties.Count, fc.Properties.Count);
Assert.AreEqual(cd.IdentityProperties.Count, fc.IdentityProperties.Count);
}
public void TestCloneProperty()
{
DataPropertyDefinition dp = new DataPropertyDefinition("Foo", "Bar");
dp.DataType = DataType.DataType_String;
dp.DefaultValue = "Whatever";
dp.IsAutoGenerated = false;
dp.Length = 45;
dp.Nullable = true;
dp.ReadOnly = false;
PropertyValueConstraintList list = new PropertyValueConstraintList();
list.ConstraintList.Add(new StringValue("A"));
list.ConstraintList.Add(new StringValue("B"));
list.ConstraintList.Add(new StringValue("C"));
dp.ValueConstraint = list;
DataPropertyDefinition dp2 = FdoSchemaUtil.CloneProperty(dp) as DataPropertyDefinition;
AssertHelper.EqualProperty(dp, dp2);
GeometricPropertyDefinition gp = new GeometricPropertyDefinition("Sna", "Fu");
gp.GeometryTypes = (int)(GeometryType.GeometryType_LineString | GeometryType.GeometryType_MultiLineString | GeometryType.GeometryType_Polygon);
gp.HasElevation = true;
gp.HasMeasure = false;
gp.ReadOnly = false;
gp.SpatialContextAssociation = "LL84";
GeometricPropertyDefinition gp2 = FdoSchemaUtil.CloneProperty(gp) as GeometricPropertyDefinition;
AssertHelper.EqualProperty(gp, gp2);
}
private bool ExportToFile(string provider, string file, int maxSchemas, int maxSpatialContexts)
{
var conf = _context.GetConfiguration();
if (maxSchemas > 0 && conf.Schemas.Count > maxSchemas)
{
ShowError("This provider only allows " + maxSchemas + " feature schema(s)");
return(false);
}
if (maxSpatialContexts > 0 && conf.SpatialContexts.Length > maxSpatialContexts)
{
ShowError("This provider only allows " + maxSpatialContexts + " spatial context(s)");
return(false);
}
if (ExpressUtility.CreateFlatFileDataSource(provider, file))
{
var conn = ExpressUtility.CreateFlatFileConnection(provider, file);
var schema = conf.Schemas[0];
schema = FdoSchemaUtil.CloneSchema(schema, true);
using (var svc = conn.CreateFeatureService())
{
IncompatibleSchema incs;
if (!svc.CanApplySchema(schema, out incs))
{
string msg = "This schema has incompatibilities. Attempt to fix it? " + incs.ToString();
if (!WrappedMessageBox.Confirm("Incompatible schema", msg, MessageBoxText.YesNo))
{
return(false);
}
schema = svc.AlterSchema(schema, incs);
}
using (var cur = new TempCursor(Cursors.WaitCursor))
{
foreach (var sc in conf.SpatialContexts)
{
svc.CreateSpatialContext(sc, true);
}
//Only for SQLite, but this is general enough
var fscs = svc.DescribeSchema();
if (fscs.Count == 1)
{
var fsc = fscs[0];
foreach (ClassDefinition cls in schema.Classes)
{
var klass = FdoSchemaUtil.CloneClass(cls);
fsc.Classes.Add(klass);
}
schema = fsc;
}
svc.ApplySchema(schema);
}
}
conn.Close();
}
return(true);
}
public void TestSchemaClone()
{
FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);
schemas.ReadXml(Path.Combine(TestHelper.CurrentPath, "Test.xml"));
FeatureSchema cloned = FdoSchemaUtil.CloneSchema(schemas[0]);
AssertHelper.EqualSchemas(schemas[0], cloned);
}
private void ApplySchemas(FdoConnection conn)
{
using (var svc = conn.CreateFeatureService())
{
using (FeatureSchemaCollection fsc = new FeatureSchemaCollection(null))
{
fsc.ReadXml(_view.SchemaFile);
int modified = FdoSchemaUtil.SetDefaultSpatialContextAssociation(fsc, "Default");
List <FeatureSchema> schemas = new List <FeatureSchema>();
if (_view.FixSchema)
{
foreach (FeatureSchema fs in fsc)
{
IncompatibleSchema inc;
if (!svc.CanApplySchema(fs, out inc))
{
FeatureSchema mod = svc.AlterSchema(fs, inc);
schemas.Add(mod);
}
else
{
schemas.Add(fs);
}
}
}
else
{
foreach (FeatureSchema fs in fsc)
{
schemas.Add(fs);
}
}
foreach (FeatureSchema fs in schemas)
{
svc.ApplySchema(fs);
}
}
}
}
public PartialSchemaSaveDialog(FeatureSchema schema)
: this()
{
_schema = FdoSchemaUtil.CloneSchema(schema); //Operate on a clone
InitTree();
}
public override IEnumerable <FdoRow> Execute(IEnumerable <FdoRow> rows)
{
if (_counter < 1) //Shouldn't be reentrant, but just play it safe.
{
/*
* Check and apply the following rules for all geometry properties to be created
*
* Target supports multiple spatial contexts:
* -------------------------------------------
* If there is no spatial contexts of the specified (source) name. Create a copy of the source spatial context.
* If there is a spatial context of the same name, using the same WKT. Do nothing
* If there is a spatial context of the same name, but using a different WKT. Create a new spatial context using the source WKT, but use a different name.
*
* Target only supports one spatial context:
* If there is no spatial context already. Create a copy of the source spatial context.
* If there is a spatial context of the same WKT. Change the association to match the name of this spatial context.
* If there is a spatial context not using the source WKT. Change the association to match the name of this spatial context. This may not be ideal, but there is no other option at this point.
*
* The regular schema compatibility fixes will handle the other properties
*/
bool targetSupportsMultipleSpatialContexts = _target.Capability.GetBooleanCapability(CapabilityType.FdoCapabilityType_SupportsMultipleSpatialContexts);
List <SpatialContextInfo> targetSpatialContexts = null;
List <SpatialContextInfo> sourceSpatialContexts = null;
if (typeof(CreateTargetClassFromSource).IsAssignableFrom(_opts.PreCopyTargetModifier.GetType()))
{
using (var tsvc = _target.CreateFeatureService())
using (var ssvc = _source.CreateFeatureService())
{
targetSpatialContexts = new List <SpatialContextInfo>(tsvc.GetSpatialContexts());
sourceSpatialContexts = new List <SpatialContextInfo>(ssvc.GetSpatialContexts());
var ct = (CreateTargetClassFromSource)_opts.PreCopyTargetModifier;
Info("Getting current schema from target");
var schema = tsvc.GetSchemaByName(_opts.TargetSchema);
if (schema.Classes.IndexOf(ct.Name) >= 0)
{
Info("Class " + _opts.TargetSchema + ":" + ct.Name + " already exists. Nothing to do here");
}
else
{
List <SpatialContextInfo> createScs = new List <SpatialContextInfo>();
var cls = ssvc.GetClassByName(ct.Schema, ct.Name);
Info("Creating a cloned copy of source class " + ct.Schema + ":" + ct.Name);
var cloned = FdoSchemaUtil.CloneClass(cls);
var propList = new List <string>(_opts.CheckSourceProperties);
var removeList = new List <string>();
foreach (PropertyDefinition prop in cloned.Properties)
{
string propName = prop.Name;
if (!propList.Contains(propName))
{
removeList.Add(propName);
}
}
if (removeList.Count > 0)
{
Info("Removing " + removeList.Count + " unused properties from cloned class");
var props = cloned.Properties;
var ids = cloned.IdentityProperties;
foreach (var name in removeList)
{
if (ids.Contains(name))
{
ids.RemoveAt(ids.IndexOf(name));
}
if (props.Contains(name))
{
props.RemoveAt(props.IndexOf(name));
}
}
Info(removeList.Count + " unused properties removed");
}
foreach (var prop in ct.PropertiesToCreate)
{
Info("Adding property to cloned class: " + prop.Name);
PropertyDefinition clonedProp = FdoSchemaUtil.CloneProperty(prop);
cloned.Properties.Add(clonedProp);
}
//Add an auto-generated identity property if none exist
if (cloned.IdentityProperties.Count == 0)
{
var id = new DataPropertyDefinition("FID", "Auto-Generated Feature Id");
id.IsAutoGenerated = true;
id.Nullable = false;
//This may not be valid for target connection, but FdoFeatureService
//will fix this for us.
id.DataType = DataType.DataType_Int32;
cloned.Properties.Add(id);
cloned.IdentityProperties.Add(id);
Info("Adding an auto-generated id (FID) to this cloned class");
}
Info("Checking this class for incompatibilities");
IncompatibleClass ic;
if (!tsvc.CanApplyClass(cloned, out ic))
{
Info("Altering this class to become compatible with target connection");
cloned = tsvc.AlterClassDefinition(cloned, ic);
Info("Class successfully altered");
}
Info("Checking if any spatial contexts need to be created and/or references modified");
foreach (PropertyDefinition pd in cloned.Properties)
{
if (pd.PropertyType == PropertyType.PropertyType_GeometricProperty)
{
AddSpatialContextsToCreate(targetSupportsMultipleSpatialContexts, targetSpatialContexts, sourceSpatialContexts, createScs, (GeometricPropertyDefinition)pd);
}
}
//We have to create spatial contexts first before applying the schema
if (createScs.Count > 0)
{
//The ones we create should be unique so no overwriting needed
ExpressUtility.CopyAllSpatialContexts(createScs, _target, false);
foreach (var sc in createScs)
{
Info("Created spatial context: " + sc.Name);
}
}
Info("Adding cloned class to target schema");
schema.Classes.Add(cloned);
Info("Applying schema back to target connection");
tsvc.ApplySchema(schema);
Info("Updated schema applied to target connection");
}
}
}
else if (typeof(UpdateTargetClass).IsAssignableFrom(_opts.PreCopyTargetModifier.GetType()))
{
var ut = (UpdateTargetClass)_opts.PreCopyTargetModifier;
using (var tsvc = _target.CreateFeatureService())
using (var ssvc = _source.CreateFeatureService())
{
targetSpatialContexts = new List <SpatialContextInfo>(tsvc.GetSpatialContexts());
sourceSpatialContexts = new List <SpatialContextInfo>(ssvc.GetSpatialContexts());
var schema = tsvc.GetSchemaByName(_opts.TargetSchema);
var cidx = schema.Classes.IndexOf(ut.Name);
if (cidx < 0)
{
throw new InvalidOperationException("Target class to be updated " + _opts.TargetSchema + ":" + ut.Name + " not found");
}
else
{
List <SpatialContextInfo> createScs = new List <SpatialContextInfo>();
var cls = schema.Classes[cidx];
foreach (var prop in ut.PropertiesToCreate)
{
if (cls.Properties.IndexOf(prop.Name) < 0)
{
Info("Adding property to class: " + prop.Name);
var clonedProp = FdoSchemaUtil.CloneProperty(prop);
if (clonedProp.PropertyType == PropertyType.PropertyType_GeometricProperty)
{
AddSpatialContextsToCreate(targetSupportsMultipleSpatialContexts, targetSpatialContexts, sourceSpatialContexts, createScs, (GeometricPropertyDefinition)clonedProp);
}
cls.Properties.Add(clonedProp);
}
else
{
Info("Skipping property " + prop.Name + " because it already exists");
}
}
//We have to create spatial contexts first before applying the schema
if (createScs.Count > 0)
{
//The ones we create should be unique so no overwriting needed
ExpressUtility.CopyAllSpatialContexts(createScs, _target, false);
foreach (var sc in createScs)
{
Info("Created spatial context: " + sc.Name);
}
}
Info("Applying modified schema " + schema.Name + " to target connection");
tsvc.ApplySchema(schema);
Info("Modified schema " + schema.Name + " applied to target connection");
}
}
}
_counter++;
}
return(rows);
}
public void TestCloneClass()
{
ClassDefinition c1 = new FeatureClass("Test", "Test Feature Class");
DataPropertyDefinition id = new DataPropertyDefinition("ID", "");
DataPropertyDefinition name = new DataPropertyDefinition("Name", "");
id.DataType = DataType.DataType_Int32;
id.IsAutoGenerated = true;
id.ReadOnly = true;
id.Nullable = false;
name.DataType = DataType.DataType_String;
name.Length = 255;
name.Nullable = true;
GeometricPropertyDefinition geom = new GeometricPropertyDefinition("Geometry", "");
geom.GeometryTypes = (int)GeometryType.GeometryType_Polygon;
geom.ReadOnly = false;
c1.Properties.Add(id);
c1.Properties.Add(name);
c1.Properties.Add(geom);
c1.IdentityProperties.Add(id);
((FeatureClass)c1).GeometryProperty = geom;
ClassDefinition cd = FdoSchemaUtil.CloneClass(c1);
Assert.IsNotNull(cd);
FeatureClass fc = cd as FeatureClass;
Assert.IsNotNull(fc);
Assert.AreEqual(cd.Name, c1.Name);
Assert.AreEqual(cd.ClassType, c1.ClassType);
Assert.IsNotNull(fc.GeometryProperty);
Assert.IsNotNull(((FeatureClass)c1).GeometryProperty);
Assert.AreEqual(fc.GeometryProperty.Name, ((FeatureClass)c1).GeometryProperty.Name);
Assert.AreEqual(cd.Description, c1.Description);
Assert.AreEqual(cd.Properties.Count, c1.Properties.Count);
Assert.AreEqual(cd.IdentityProperties.Count, c1.IdentityProperties.Count);
c1.Dispose();
c1 = new Class("TestClass", "Test Class");
id = new DataPropertyDefinition("ID", "");
name = new DataPropertyDefinition("Name", "");
id.DataType = DataType.DataType_Int32;
id.IsAutoGenerated = true;
id.ReadOnly = true;
id.Nullable = false;
name.DataType = DataType.DataType_String;
name.Length = 255;
name.Nullable = true;
cd = FdoSchemaUtil.CloneClass(c1);
Assert.IsNotNull(cd);
Assert.AreEqual(cd.Name, c1.Name);
Assert.AreEqual(cd.Description, c1.Description);
Assert.AreEqual(cd.Properties.Count, c1.Properties.Count);
Assert.AreEqual(cd.IdentityProperties.Count, c1.IdentityProperties.Count);
}
internal static FdoClassCopyOptions FromElement(FdoCopyTaskElement el, FeatureSchemaCache cache, FdoConnection sourceConn, FdoConnection targetConn, out TargetClassModificationItem mod)
{
mod = null;
if (!cache.HasConnection(el.Source.connection))
{
throw new TaskLoaderException("The referenced source connection is not defined");
}
if (!cache.HasConnection(el.Target.connection))
{
throw new TaskLoaderException("The referenced target connection is not defined");
}
FdoClassCopyOptions opts = new FdoClassCopyOptions(el.Source.connection, el.Target.connection, el.Source.schema, el.Source.@class, el.Target.schema, el.Target.@class);
opts.DeleteTarget = el.Options.DeleteTarget;
opts.SourceFilter = el.Options.Filter;
if (!el.Options.FlattenGeometriesSpecified)
{
opts.FlattenGeometries = false;
}
else
{
opts.FlattenGeometries = el.Options.FlattenGeometries;
}
if (!el.Options.ForceWKBSpecified)
{
opts.ForceWkb = false;
}
else
{
opts.ForceWkb = el.Options.ForceWKB;
}
if (!string.IsNullOrEmpty(el.Options.BatchSize))
{
opts.BatchSize = Convert.ToInt32(el.Options.BatchSize);
}
opts.Name = el.name;
opts.CreateIfNotExists = el.createIfNotExists;
ClassDefinition srcClass = cache.GetClassByName(el.Source.connection, el.Source.schema, el.Source.@class);
ClassDefinition dstClass = cache.GetClassByName(el.Target.connection, el.Target.schema, el.Target.@class);
if (!el.createIfNotExists && dstClass == null)
{
throw new InvalidOperationException("Target class " + el.Target.@class + " does not exist and the createIfNotExist option is false");
}
SpatialContextInfo defaultSc = null;
FunctionDefinitionCollection availableFunctions = (FunctionDefinitionCollection)sourceConn.Capability.GetObjectCapability(CapabilityType.FdoCapabilityType_ExpressionFunctions);
using (var svc = targetConn.CreateFeatureService())
{
defaultSc = svc.GetActiveSpatialContext();
}
if (dstClass != null)
{
foreach (FdoPropertyMappingElement propMap in el.PropertyMappings)
{
if (srcClass.Properties.IndexOf(propMap.source) < 0)
{
throw new TaskLoaderException("The property mapping (" + propMap.source + " -> " + propMap.target + ") in task (" + el.name + ") contains a source property not found in the source class definition (" + el.Source.@class + ")");
}
//Add to list of properties to check for
if (propMap.createIfNotExists && dstClass.Properties.IndexOf(propMap.target) < 0)
{
if (mod == null)
{
mod = new UpdateTargetClass(dstClass.Name);
}
opts.AddSourcePropertyToCheck(propMap.source);
//Clone copy of source property of same name
var srcProp = srcClass.Properties[srcClass.Properties.IndexOf(propMap.source)];
srcProp = FdoSchemaUtil.CloneProperty(srcProp);
mod.AddProperty(srcProp);
}
else
{
if (dstClass.Properties.IndexOf(propMap.target) < 0)
{
throw new TaskLoaderException("The property mapping (" + propMap.source + " -> " + propMap.target + ") in task (" + el.name + ") contains a target property not found in the target class definition (" + el.Target.@class + ")");
}
PropertyDefinition sp = srcClass.Properties[propMap.source];
PropertyDefinition tp = dstClass.Properties[propMap.target];
if (sp.PropertyType != tp.PropertyType)
{
throw new TaskLoaderException("The properties in the mapping (" + propMap.source + " -> " + propMap.target + ") are of different types");
}
//if (sp.PropertyType != PropertyType.PropertyType_DataProperty)
// throw new TaskLoaderException("One or more properties in the mapping (" + propMap.source + " -> " + propMap.target + ") is not a data property");
DataPropertyDefinition sdp = sp as DataPropertyDefinition;
DataPropertyDefinition tdp = tp as DataPropertyDefinition;
opts.AddPropertyMapping(propMap.source, propMap.target);
//Property mapping is between two data properties
if (sdp != null && tdp != null)
{
//Types not equal, so add a conversion rule
if (sdp.DataType != tdp.DataType)
{
FdoDataPropertyConversionRule rule = new FdoDataPropertyConversionRule(
propMap.source,
propMap.target,
sdp.DataType,
tdp.DataType,
propMap.nullOnFailedConversion,
propMap.truncate);
opts.AddDataConversionRule(propMap.source, rule);
}
}
}
}
//
foreach (FdoExpressionMappingElement exprMap in el.ExpressionMappings)
{
if (string.IsNullOrEmpty(exprMap.target))
{
continue;
}
opts.AddSourceExpression(exprMap.alias, exprMap.Expression, exprMap.target);
//Add to list of properties to check for
if (exprMap.createIfNotExists)
{
//Class exists but property doesn't
if (dstClass.Properties.IndexOf(exprMap.target) < 0)
{
if (mod == null)
{
mod = new UpdateTargetClass(el.Target.@class);
}
var prop = FdoSchemaUtil.CreatePropertyFromExpressionType(exprMap.Expression, srcClass, availableFunctions, defaultSc.Name);
if (prop == null)
{
throw new InvalidOperationException("Could not derive a property definition from the expression: " + exprMap.Expression);
}
prop.Name = exprMap.target;
mod.AddProperty(prop);
}
}
else //Conversion rules can only apply if both properties exist.
{
FdoPropertyType?pt = ExpressionUtility.ParseExpressionType(exprMap.Expression, sourceConn);
if (pt.HasValue)
{
DataType?srcDt = ValueConverter.GetDataType(pt.Value);
if (srcDt.HasValue)
{
PropertyDefinition tp = dstClass.Properties[exprMap.target];
DataPropertyDefinition tdp = tp as DataPropertyDefinition;
if (tdp != null)
{
if (srcDt.Value != tdp.DataType)
{
FdoDataPropertyConversionRule rule = new FdoDataPropertyConversionRule(
exprMap.alias,
exprMap.target,
srcDt.Value,
tdp.DataType,
exprMap.nullOnFailedConversion,
exprMap.truncate);
opts.AddDataConversionRule(exprMap.alias, rule);
}
}
}
}
}
}
}
else //class doesn't exist
{
mod = new CreateTargetClassFromSource(el.Source.schema, el.Target.@class);
foreach (var propMap in el.PropertyMappings)
{
opts.AddPropertyMapping(propMap.source, propMap.target);
if (propMap.createIfNotExists)
{
opts.AddSourcePropertyToCheck(propMap.source);
}
}
foreach (var exprMap in el.ExpressionMappings)
{
opts.AddSourceExpression(exprMap.alias, exprMap.Expression, exprMap.target);
if (exprMap.createIfNotExists)
{
opts.AddSourcePropertyToCheck(exprMap.alias);
}
var prop = FdoSchemaUtil.CreatePropertyFromExpressionType(exprMap.Expression, srcClass, availableFunctions, defaultSc.Name);
if (prop == null)
{
throw new InvalidOperationException("Could not derive a property definition from the expression: " + exprMap.Expression);
}
prop.Name = exprMap.target;
mod.AddProperty(prop);
}
}
return(opts);
}
/// <summary>
/// Initializes this instance.
/// </summary>
protected override void Initialize()
{
var sw = new Stopwatch();
sw.Start();
_options.Validate();
SendMessage("Setting up left and right sides of the join");
// Abstract:
//
// The current built-in join infrastructure is very naive as it uses nested
// loops. Being a O(m*n) operation, the slowdown becomes readily apparent
// as the size of the data you're working with increases. As such, the
// current infrastructure is woefully inadequate for large datasets.
//
// How can we fix this problem? We could try to implement various join
// algorithms for different scenarios, which would be a laborious exercise
// in itself.
//
// Or, we can just delegate this problem to the universal swiss army-knife of
// databases, SQLite.
//
// SQLite has many things going for it, including:
// - Support for common types of joins (important!)
// - LIGHTING FAST insert performance. The current FdoInputOperation is already optimised for SQLite
// - LIGHTING FAST read performance
// - Ability to use SQL to modify the database internals, such as creating indexes (FDO provider supports SQL commands)
//
// As such, SQLite is the perfect candidate for a temp data store to merge two
// disparate data sources. The time spent setting up this temp SQLite database (ie. Copying "left" and
// "right" side data into it) is negligible in the grand scheme of things.
//
// Process Overview:
//
// 1. Create temp SQLite database
// 2. Pump left and right sides into this database
// 3. Create indexes on join columns of both tables (IMPORTANT)
// 4. Create a view encapsulating our join
// 5. Copy this view out to our target
//
// Additional Notes:
//
// We will have to change our supported join types to line up with what SQLite supports, which
// are:
// - INNER JOIN
// - LEFT OUTER JOIN
//
// SQLite does not support RIGHT OUTER JOINs but these could be emulated by inverting the
// "left" and "right" tables for the LEFT OUTER JOIN. FULL OUTER JOIN is not supporte by
// SQLite so this will be removed from our API.
//
// Since this SQLite database is temporary, we don't bother with putting
// the right spatial context in there. Spatial contexts do not (should not) affect
// the underlying coordinates of any geometries moving to and from the data store.
//
// SQLite views by default are represented as non-Feature classes. Geometry properties
// default to BLOB data types. To "fix" this we need to add a new entry to the geometry_columns
// metadata table. This may produce an incorrect feature class (ie. Has 1-n geometry properties
// but no designated one), this is okay as we only care that the properties are there and the
// temp-target property mappings check out.
//
// Although the implementation will change, the requirements remain the same, which are:
//
// 1. The target class must not already exist (as it will be created)
// 2. If no designated geometry is specified, then the class definition will be FdoClass and not FdoFeatureClass
ClassDefinition leftCls = null;
ClassDefinition rightCls = null;
ClassDefinition mergedCls = null;
using (var leftSvc = _options.Left.Connection.CreateFeatureService())
using (var rightSvc = _options.Right.Connection.CreateFeatureService())
{
leftCls = leftSvc.GetClassByName(_options.Left.SchemaName, _options.Left.ClassName);
rightCls = rightSvc.GetClassByName(_options.Right.SchemaName, _options.Right.ClassName);
if (leftCls == null)
{
throw new FdoETLException("Left class not found " + _options.Left.SchemaName + ":" + _options.Left.ClassName);
}
if (rightCls == null)
{
throw new FdoETLException("Right class not found " + _options.Right.SchemaName + ":" + _options.Right.ClassName);
}
var leftJoinProps = new List <string>(_options.JoinPairs.AllKeys);
var rightJoinProps = new List <string>();
foreach (var p in leftJoinProps)
{
rightJoinProps.Add(_options.JoinPairs[p]);
}
var leftGeom = (!string.IsNullOrEmpty(_options.GeometryProperty) && _options.Side == JoinSide.Left) ? _options.GeometryProperty : null;
var rightGeom = (!string.IsNullOrEmpty(_options.GeometryProperty) && _options.Side == JoinSide.Right) ? _options.GeometryProperty : null;
PrepareClass(leftCls, _options.LeftProperties, leftJoinProps, _options.LeftPrefix, leftGeom);
PrepareClass(rightCls, _options.RightProperties, rightJoinProps, _options.RightPrefix, rightGeom);
mergedCls = CreateMergedClass(leftCls, rightCls);
}
var dprops = new NameValueCollection();
dprops["File"] = Path.GetTempFileName();
var tempSchema = new FeatureSchema("Default", "");
var leftCopy = FdoSchemaUtil.CloneClass(leftCls);
var rightCopy = FdoSchemaUtil.CloneClass(rightCls);
string leftClassName = "LEFT_SIDE";
string rightClassName = "RIGHT_SIDE";
leftCopy.Name = leftClassName;
rightCopy.Name = rightClassName;
tempSchema.Classes.Add(leftCopy);
tempSchema.Classes.Add(rightCopy);
//Create SQLite database
Register(new FdoCreateDataStoreOperation("OSGeo.SQLite", dprops, null));
//Apply temp schema
var tempConn = new FdoConnection("OSGeo.SQLite", "File=" + dprops["File"]);
Register(new FdoApplySchemaOperation(tempConn, tempSchema));
#if DEBUG
Register(new FdoSingleActionOperation(() => { SendMessage("Temp db created in: " + dprops["File"]); }));
#endif
//Prep property mappings for bulk copy
var leftMaps = new NameValueCollection();
var rightMaps = new NameValueCollection();
var leftQuery = new FeatureQueryOptions(leftCls.Name);
var rightQuery = new FeatureQueryOptions(rightCls.Name);
foreach (var leftp in _options.LeftProperties)
{
if (string.IsNullOrEmpty(_options.LeftPrefix))
{
leftMaps.Add(leftp, leftp);
}
else
{
leftMaps.Add(leftp, _options.LeftPrefix + leftp);
}
leftQuery.AddFeatureProperty(leftp);
}
foreach (var rightp in _options.RightProperties)
{
if (string.IsNullOrEmpty(_options.RightPrefix))
{
rightMaps.Add(rightp, rightp);
}
else
{
rightMaps.Add(rightp, _options.RightPrefix + rightp);
}
rightQuery.AddFeatureProperty(rightp);
}
if (!string.IsNullOrEmpty(_options.LeftFilter))
{
leftQuery.Filter = _options.LeftFilter;
}
if (!string.IsNullOrEmpty(_options.RightFilter))
{
rightQuery.Filter = _options.RightFilter;
}
//don't forget join keys
foreach (string l in _options.JoinPairs.Keys)
{
string r = _options.JoinPairs[l];
if (!_options.LeftProperties.Contains(l))
{
leftQuery.AddFeatureProperty(l);
if (string.IsNullOrEmpty(_options.LeftPrefix))
{
leftMaps.Add(l, l);
}
else
{
leftMaps.Add(l, _options.LeftPrefix + l);
}
}
if (!_options.RightProperties.Contains(r))
{
rightQuery.AddFeatureProperty(r);
if (string.IsNullOrEmpty(_options.RightPrefix))
{
rightMaps.Add(r, r);
}
else
{
rightMaps.Add(r, _options.RightPrefix + r);
}
}
}
//don't forget geometry!
if (!string.IsNullOrEmpty(_options.GeometryProperty))
{
if (_options.Side == JoinSide.Left)
{
if (!leftQuery.PropertyList.Contains(_options.GeometryProperty))
{
leftQuery.AddFeatureProperty(_options.GeometryProperty);
if (string.IsNullOrEmpty(_options.LeftPrefix))
{
leftMaps.Add(_options.GeometryProperty, _options.GeometryProperty);
}
else
{
leftMaps.Add(_options.GeometryProperty, _options.LeftPrefix + _options.GeometryProperty);
}
}
}
else
{
if (!rightQuery.PropertyList.Contains(_options.GeometryProperty))
{
rightQuery.AddFeatureProperty(_options.GeometryProperty);
if (string.IsNullOrEmpty(_options.RightPrefix))
{
rightMaps.Add(_options.GeometryProperty, _options.GeometryProperty);
}
else
{
rightMaps.Add(_options.GeometryProperty, _options.RightPrefix + _options.GeometryProperty);
}
}
}
}
var copyLeftErrors = new List <Exception>();
var copyRightErrors = new List <Exception>();
var copyTargetErrors = new List <Exception>();
//Copy left source
ParameterlessAction copyLeft = () =>
{
SendMessage("Copying left source with filter: " + _options.LeftFilter);
var copy = ExpressUtility.CreateBulkCopy(
_options.Left.Connection,
tempConn,
_options.Left.SchemaName,
leftQuery,
tempSchema.Name, //temp sqlite schema name
leftClassName, //sqlite "left" class name
leftMaps);
copy.ProcessMessage += delegate(object sender, MessageEventArgs e)
{
SendMessage(e.Message);
};
copy.Execute();
copyLeftErrors.AddRange(copy.GetAllErrors());
};
Register(new FdoSingleActionOperation(copyLeft));
//Register(new FdoInputOperation(_options.Left.Connection, leftQuery));
//Register(new FdoOutputOperation(tempConn, leftClassName, leftMaps));
//Copy right source
ParameterlessAction copyRight = () =>
{
SendMessage("Copying right source with filter: " + _options.RightFilter);
var copy = ExpressUtility.CreateBulkCopy(
_options.Right.Connection,
tempConn,
_options.Right.SchemaName,
rightQuery,
tempSchema.Name, //temp sqlite schema name
rightClassName, //sqlite "right" class name
rightMaps);
copy.ProcessMessage += delegate(object sender, MessageEventArgs e)
{
SendMessage(e.Message);
};
copy.Execute();
copyRightErrors.AddRange(copy.GetAllErrors());
};
Register(new FdoSingleActionOperation(copyRight));
//Register(new FdoInputOperation(_options.Right.Connection, rightQuery));
//Register(new FdoOutputOperation(tempConn, rightClassName, rightMaps));
string srcClass = "VIEW_INPUT";
//Create indexes on left and right sides to optimize read performance
ParameterlessAction indexLeft = () =>
{
using (var svc = tempConn.CreateFeatureService())
{
SendMessage("Creating left side index in temp db");
string sql = "CREATE INDEX IDX_LEFT_ID ON " + leftClassName + "(";
var tokens = new List <string>();
foreach (string p in _options.JoinPairs.Keys)
{
if (!string.IsNullOrEmpty(_options.LeftPrefix))
{
tokens.Add(_options.LeftPrefix + p);
}
else
{
tokens.Add(p);
}
}
sql = sql + string.Join(", ", tokens.ToArray()) + ")";
SendMessage("Executing SQL: " + sql);
svc.ExecuteSQLNonQuery(sql);
}
};
ParameterlessAction indexRight = () =>
{
using (var svc = tempConn.CreateFeatureService())
{
SendMessage("Creating right side index in temp db");
string sql = "CREATE INDEX IDX_RIGHT_ID ON " + rightClassName + "(";
var tokens = new List <string>();
foreach (string p in _options.JoinPairs.Keys)
{
string prop = _options.JoinPairs[p];
if (!string.IsNullOrEmpty(_options.RightPrefix))
{
tokens.Add(_options.RightPrefix + prop);
}
else
{
tokens.Add(prop);
}
}
sql = sql + string.Join(", ", tokens.ToArray()) + ")";
SendMessage("Executing SQL: " + sql);
svc.ExecuteSQLNonQuery(sql);
}
};
Register(new FdoSingleActionOperation(indexLeft));
Register(new FdoSingleActionOperation(indexRight));
//Create view
ParameterlessAction createView = () =>
{
using (var svc = tempConn.CreateFeatureService())
{
SendMessage("Creating view in temp db");
StringBuilder sql = new StringBuilder("CREATE VIEW ");
sql.Append(srcClass + " AS SELECT ");
foreach (var p in _options.LeftProperties)
{
if (!string.IsNullOrEmpty(_options.LeftPrefix))
{
sql.Append("l." + _options.LeftPrefix + p + ", ");
}
else
{
sql.Append("l." + p + ", ");
}
}
if (!string.IsNullOrEmpty(_options.GeometryProperty))
{
if (_options.Side == JoinSide.Left)
{
if (!_options.LeftProperties.Contains(_options.GeometryProperty))
{
if (!string.IsNullOrEmpty(_options.LeftPrefix))
{
sql.Append("l." + _options.LeftPrefix + _options.GeometryProperty + ", ");
}
else
{
sql.Append("l." + _options.GeometryProperty + ", ");
}
}
}
else
{
if (!_options.RightProperties.Contains(_options.GeometryProperty))
{
if (!string.IsNullOrEmpty(_options.RightPrefix))
{
sql.Append("r." + _options.RightPrefix + _options.GeometryProperty + ", ");
}
else
{
sql.Append("r." + _options.GeometryProperty + ", ");
}
}
}
}
int rc = _options.RightProperties.Count;
int i = 0;
foreach (var p in _options.RightProperties)
{
string pn = p;
if (!string.IsNullOrEmpty(_options.RightPrefix))
{
pn = _options.RightPrefix + pn;
}
if (i == rc - 1)
{
sql.Append("r." + pn + " FROM ");
}
else
{
sql.Append("r." + pn + ", ");
}
i++;
}
sql.Append(leftClassName + " l ");
switch (_options.JoinType)
{
case FdoJoinType.Inner:
sql.Append("INNER JOIN " + rightClassName + " r ON ");
break;
case FdoJoinType.Left:
sql.Append("LEFT OUTER JOIN " + rightClassName + " r ON ");
break;
default:
throw new FdoETLException("Unsupported join type: " + _options.JoinType);
}
rc = _options.JoinPairs.Count;
i = 0;
foreach (string l in _options.JoinPairs.Keys)
{
string r = _options.JoinPairs[l];
string left = l;
string right = r;
if (!string.IsNullOrEmpty(_options.LeftPrefix))
{
left = _options.LeftPrefix + left;
}
if (!string.IsNullOrEmpty(_options.RightPrefix))
{
right = _options.RightPrefix + right;
}
if (i == rc - 1)
{
sql.Append("l." + left + " = r." + right);
}
else
{
sql.Append("l." + left + " = r." + right + " AND ");
}
i++;
}
SendMessage("Executing SQL: " + sql.ToString());
svc.ExecuteSQLNonQuery(sql.ToString());
}
};
Register(new FdoSingleActionOperation(createView));
//Hack FDO metadata to make this a feature class
if (!string.IsNullOrEmpty(_options.GeometryProperty))
{
ParameterlessAction reg = () =>
{
using (var svc = tempConn.CreateFeatureService())
{
SendMessage("Exposing view as a FDO feature class");
string sql = "INSERT INTO geometry_columns(f_table_name, f_geometry_column, geometry_type, geometry_dettype, coord_dimension, srid, geometry_format) VALUES('" + srcClass + "','" + _options.GeometryProperty + "',15,7743,0,0,'FGF')";
SendMessage("Executing SQL: " + sql.ToString());
svc.ExecuteSQLNonQuery(sql);
}
};
Register(new FdoSingleActionOperation(reg));
}
//Copy view to target
ParameterlessAction applyTarget = () =>
{
using (var svc = _options.Target.Connection.CreateFeatureService())
{
SendMessage("Fetching target schema");
var schema = svc.GetSchemaByName(_options.Target.SchemaName);
IncompatibleClass cls;
if (!svc.CanApplyClass(mergedCls, out cls))
{
SendMessage("Fixing incompatibilities in merged class");
mergedCls = svc.AlterClassDefinition(mergedCls, cls);
}
SendMessage("Adding merged class to target schema");
schema.Classes.Add(mergedCls);
SendMessage("Applying modified target schema");
svc.ApplySchema(schema);
}
};
Register(new FdoSingleActionOperation(applyTarget));
var tempQuery = new FeatureQueryOptions("VIEW_INPUT");
var targetMapping = new NameValueCollection();
foreach (PropertyDefinition p in mergedCls.Properties)
{
tempQuery.AddFeatureProperty(p.Name);
//Target class is a replica of the temp one, so all properties
//have the same name in both source and target
targetMapping[p.Name] = p.Name;
}
ParameterlessAction copyToTarget = () =>
{
var copy = ExpressUtility.CreateBulkCopy(
tempConn,
_options.Target.Connection,
tempSchema.Name,
tempQuery,
_options.Target.SchemaName,
_options.Target.ClassName,
targetMapping);
copy.ProcessMessage += delegate(object sender, MessageEventArgs e)
{
SendMessage(e.Message);
};
copy.Execute();
copyTargetErrors.AddRange(copy.GetAllErrors());
sw.Stop();
};
Register(new FdoSingleActionOperation(copyToTarget));
//Log all errors
ParameterlessAction logErrors = () =>
{
SendMessage(copyLeftErrors.Count + " errors encountered copying left source to temp db");
_allErrors.AddRange(copyLeftErrors);
SendMessage(copyRightErrors.Count + " errors encountered copying right source to temp db");
_allErrors.AddRange(copyRightErrors);
SendMessage(copyTargetErrors.Count + " errors encountered copying merged source to target");
_allErrors.AddRange(copyTargetErrors);
SendMessage("Join Operation completed in " + sw.Elapsed.ToString());
};
Register(new FdoSingleActionOperation(logErrors));
}
private ClassDefinition CreateMergedClass(ClassDefinition leftCls, ClassDefinition rightCls)
{
ClassDefinition cls = null;
if (!string.IsNullOrEmpty(_options.GeometryProperty))
{
cls = new FeatureClass(_options.Target.ClassName, "");
}
else
{
cls = new Class(_options.Target.ClassName, "");
}
var props = cls.Properties;
foreach (PropertyDefinition p in leftCls.Properties)
{
int idx = props.IndexOf(p.Name);
if (idx < 0)
{
var prop = FdoSchemaUtil.CloneProperty(p);
props.Add(prop);
}
}
foreach (PropertyDefinition p in rightCls.Properties)
{
int idx = props.IndexOf(p.Name);
if (idx < 0)
{
var prop = FdoSchemaUtil.CloneProperty(p);
props.Add(prop);
}
}
//Strip off autogeneration because we want to preserve original values
foreach (PropertyDefinition p in props)
{
if (p.PropertyType == PropertyType.PropertyType_DataProperty)
{
DataPropertyDefinition dp = (DataPropertyDefinition)p;
dp.IsAutoGenerated = false;
}
}
DataPropertyDefinition fid = new DataPropertyDefinition("FID", "Autogenerated ID");
fid.DataType = DataType.DataType_Int32;
fid.IsAutoGenerated = true;
fid.Nullable = false;
props.Add(fid);
cls.IdentityProperties.Add(fid);
if (!string.IsNullOrEmpty(_options.GeometryProperty))
{
//If prefixed, we need to qualify it to match what's in the merged class
string pn = _options.GeometryProperty;
if (_options.Side == JoinSide.Left)
{
if (!string.IsNullOrEmpty(_options.LeftPrefix))
{
pn = _options.LeftPrefix + pn;
}
}
else
{
if (!string.IsNullOrEmpty(_options.RightPrefix))
{
pn = _options.RightPrefix + pn;
}
}
int idx = props.IndexOf(pn);
if (idx < 0)
{
throw new FdoETLException("Property not found in merged class: " + _options.GeometryProperty);
}
else
{
var p = props[idx];
if (p.PropertyType != PropertyType.PropertyType_GeometricProperty)
{
throw new FdoETLException("Designated property is not a geometry property: " + _options.GeometryProperty);
}
((FeatureClass)cls).GeometryProperty = (GeometricPropertyDefinition)p;
}
}
return(cls);
}