private void Load(FdoBulkCopyOptions def, string name)
{
txtName.Text = name;
grdConnections.Rows.Clear();
foreach (string connName in def.ConnectionNames)
{
this.AddParticipatingConnection(connName);
}
TreeNode root = mTreeView.Nodes[0];
foreach (FdoClassCopyOptions task in def.ClassCopyOptions)
{
//Init w/ defaults
CopyTaskNodeDecorator dec = AddNewTask(
root,
task.SourceConnectionName,
task.SourceSchema,
task.SourceClassName,
task.TargetConnectionName,
task.TargetSchema,
task.TargetClassName,
task.Name,
task.CreateIfNotExists);
_tasks[dec.DecoratedNode.Index] = dec;
root.Expand();
btnSave.Enabled = (root.Nodes.Count > 0);
//Options
dec.Options.BatchSize = task.BatchSize;
dec.Options.Delete = task.DeleteTarget;
dec.Options.SourceFilter = task.SourceFilter;
dec.Options.Flatten = task.FlattenGeometries;
var checkProps = new List<string>(task.CheckSourceProperties);
//Property Mappings
foreach (string srcProp in task.SourcePropertyNames)
{
string dstProp = task.GetTargetProperty(srcProp);
bool createIfNotExists = checkProps.Contains(srcProp);
try
{
dec.PropertyMappings.MapProperty(srcProp, dstProp, createIfNotExists);
}
catch (MappingException ex)
{
LoggingService.Info("Skipping mapping: " + srcProp + " => " + dstProp + " (" + ex.Message + ")");
}
FdoDataPropertyConversionRule rule = task.GetDataConversionRule(srcProp);
PropertyConversionNodeDecorator cd = dec.PropertyMappings.GetConversionRule(srcProp);
if (rule != null)
{
cd.NullOnFailedConversion = rule.NullOnFailure;
cd.Truncate = rule.Truncate;
}
}
//Expression Mappings
foreach (string alias in task.SourceAliases)
{
string expr = task.GetExpression(alias);
string dstProp = task.GetTargetPropertyForAlias(alias);
bool createIfNotExists = checkProps.Contains(alias);
dec.ExpressionMappings.AddExpression(alias, expr);
dec.ExpressionMappings.MapExpression(alias, dstProp, createIfNotExists);
FdoDataPropertyConversionRule rule = task.GetDataConversionRule(alias);
PropertyConversionNodeDecorator cd = dec.ExpressionMappings.GetConversionRule(alias);
if (rule != null)
{
cd.NullOnFailedConversion = rule.NullOnFailure;
cd.Truncate = rule.Truncate;
}
}
}
}
public override int Execute()
{
CommandStatus retCode;
FdoConnection srcConn = new FdoConnection(_srcProvider, _srcConnStr);
FdoConnection destConn = null;
//Directory given, assume SHP
if (Directory.Exists(_destPath))
{
destConn = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + _destPath);
}
else
{
if (ExpressUtility.CreateFlatFileDataSource(_destPath))
destConn = ExpressUtility.CreateFlatFileConnection(_destPath);
else
throw new FdoException("Could not create data source: " + _destPath);
}
try
{
srcConn.Open();
destConn.Open();
string srcName = "SOURCE";
string dstName = "TARGET";
FdoBulkCopyOptions options = new FdoBulkCopyOptions();
options.RegisterConnection(srcName, srcConn);
options.RegisterConnection(dstName, destConn);
using (FdoFeatureService srcService = srcConn.CreateFeatureService())
using (FdoFeatureService destService = destConn.CreateFeatureService())
{
//See if spatial context needs to be copied to target
if (!string.IsNullOrEmpty(_srcSpatialContext))
{
SpatialContextInfo srcCtx = srcService.GetSpatialContext(_srcSpatialContext);
if (srcCtx != null)
{
Console.WriteLine("Copying spatial context: " + srcCtx.Name);
ExpressUtility.CopyAllSpatialContexts(new SpatialContextInfo[] { srcCtx }, destConn, true);
}
}
else
{
//Copy all
ExpressUtility.CopyAllSpatialContexts(srcConn, destConn, true);
}
FeatureSchema srcSchema = null;
//See if partial class list is needed
if (_srcClasses.Count > 0)
{
WriteLine("Checking if partial schema discovery is supported: " + srcService.SupportsPartialSchemaDiscovery());
srcSchema = srcService.PartialDescribeSchema(_srcSchema, _srcClasses);
}
else //Full copy
{
WriteLine("No classes specified, reading full source schema");
srcSchema = srcService.GetSchemaByName(_srcSchema);
}
if (srcSchema == null)
{
WriteError("Could not find source schema: " + _srcSchema);
retCode = CommandStatus.E_FAIL_SCHEMA_NOT_FOUND;
}
else
{
WriteLine("Checking source schema for incompatibilities");
FeatureSchema targetSchema = null;
IncompatibleSchema incSchema;
if (destService.CanApplySchema(srcSchema, out incSchema))
{
int clsCount = srcSchema.Classes.Count;
WriteLine("Applying source schema (containing " + clsCount + " classes) to target");
destService.ApplySchema(srcSchema, null, true);
targetSchema = srcSchema;
}
else
{
WriteWarning("Incompatibilities were detected in source schema. Applying a modified version to target");
FeatureSchema fixedSchema = destService.AlterSchema(srcSchema, incSchema);
int clsCount = fixedSchema.Classes.Count;
WriteLine("Applying modified source schema (containing " + clsCount + " classes) to target");
destService.ApplySchema(fixedSchema, null, true);
targetSchema = fixedSchema;
}
//Now set class copy options
foreach (ClassDefinition cd in srcSchema.Classes)
{
FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, srcSchema.Name, cd.Name, targetSchema.Name, cd.Name);
copt.FlattenGeometries = _flatten;
options.AddClassCopyOption(copt);
}
if (_flatten)
{
WriteWarning("The switch -flatten has been defined. Geometries that are copied will have any Z or M coordinates removed");
}
FdoBulkCopy copy = new FdoBulkCopy(options);
copy.ProcessMessage += new MessageEventHandler(OnMessage);
copy.ProcessCompleted += new EventHandler(OnCompleted);
Console.WriteLine("Executing bulk copy");
copy.Execute();
List<Exception> errors = new List<Exception>(copy.GetAllErrors());
if (errors.Count > 0)
{
string file = GenerateLogFileName("bcp-error-");
LogErrors(errors, file);
base.WriteError("Errors were encountered during bulk copy.");
retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS;
}
else { retCode = CommandStatus.E_OK; }
retCode = CommandStatus.E_OK;
}
}
}
catch (Exception ex)
{
WriteException(ex);
retCode = CommandStatus.E_FAIL_UNKNOWN;
}
finally
{
srcConn.Dispose();
destConn.Dispose();
}
return (int)retCode;
}
/// <summary>
/// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
/// </summary>
/// <param name="options">The options.</param>
public FdoBulkCopy(FdoBulkCopyOptions options)
{
_options = options;
}
/// <summary>
/// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
/// </summary>
/// <param name="options">The options.</param>
/// <param name="reportFrequency">The report frequency.</param>
public FdoBulkCopy(FdoBulkCopyOptions options, int reportFrequency)
: this(options)
{
_ReportFrequency = reportFrequency;
}
/// <summary>
/// Utility method to create a bulk copy operation from
/// one class to another
/// </summary>
/// <param name="sourceConn"></param>
/// <param name="targetConn"></param>
/// <param name="srcSchemaName"></param>
/// <param name="srcQuery"></param>
/// <param name="targetSchemaName"></param>
/// <param name="targetClassName"></param>
/// <param name="propertyMapping"></param>
/// <returns></returns>
public static FdoBulkCopy CreateBulkCopy(
FdoConnection sourceConn,
FdoConnection targetConn,
string srcSchemaName,
FeatureQueryOptions srcQuery,
string targetSchemaName,
string targetClassName,
NameValueCollection propertyMapping)
{
var dict = new Dictionary<string, FdoConnection>();
dict["SOURCE"] = sourceConn;
dict["TARGET"] = targetConn;
var opts = new FdoBulkCopyOptions(dict, false);
var copt = new FdoClassCopyOptions(
"SOURCE",
"TARGET",
srcSchemaName,
srcQuery.ClassName,
targetSchemaName,
targetClassName);
if (!string.IsNullOrEmpty(srcQuery.Filter))
copt.SourceFilter = srcQuery.Filter;
foreach (string p in propertyMapping.Keys)
{
copt.AddPropertyMapping(p, propertyMapping[p]);
}
copt.FlattenGeometries = true;
copt.ForceWkb = true;
opts.AddClassCopyOption(copt);
return new FdoBulkCopy(opts, 100);
}
/// <summary>
/// Creates a FDO bulk copy task. The target file will be created as part of
/// this method call. If the target path is a directory, it is assumed that
/// SHP files are to be created and copied to.
/// </summary>
/// <param name="sourceFile">The path to the source file.</param>
/// <param name="targetPath">
/// The path to the target file/directory. If it is a directory, it is assumed
/// that SHP files are to be created and copied to.
/// </param>
/// <param name="copySpatialContexts">If true, will also copy spatial contexts</param>
/// <param name="fixIncompatibleSchema">If true, will try to fix the source schema to make it compatible with the target connection. If false, an exception will be thrown</param>
/// <param name="flattenGeometries">If true, will strip all Z and M coordinates from all geometries that are copied</param>
/// <returns></returns>
public static FdoBulkCopy CreateBulkCopy(string sourceFile, string targetPath, bool copySpatialContexts, bool fixIncompatibleSchema, bool flattenGeometries)
{
FdoBulkCopyOptions options = null;
FdoConnection source = null;
FdoConnection target = null;
try
{
//Is a directory. Implies a SHP connection
if (IsShp(targetPath))
{
//SHP doesn't actually support CreateDataStore. We use the following technique:
// - Connect to base directory
// - Clone source schema and apply to SHP connection.
// - A SHP file and related files are created for each feature class.
string shpdir = Directory.Exists(targetPath) ? targetPath : Path.GetDirectoryName(targetPath);
source = CreateFlatFileConnection(sourceFile);
target = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + shpdir);
source.Open();
//Verify source has only classes with single geometry storage and only one geometry
using (FdoFeatureService svc = source.CreateFeatureService())
{
using (FeatureSchemaCollection schemas = svc.DescribeSchema())
{
foreach (FeatureSchema sch in schemas)
{
foreach (ClassDefinition cd in sch.Classes)
{
int geomProps = 0;
foreach (PropertyDefinition pd in cd.Properties)
{
if (pd.PropertyType == PropertyType.PropertyType_GeometricProperty)
{
GeometricPropertyDefinition gp = pd as GeometricPropertyDefinition;
GeometricType[] types = FdoGeometryUtil.GetGeometricTypes(gp.GeometryTypes);
if (types.Length != 1 || (types.Length == 1 && types[0] == GeometricType.GeometricType_All))
throw new FdoETLException(string.Format("Source file cannot be copied to a SHP file. {0}:{1}.{2} has more than one geometry storage type", sch.Name, cd.Name, pd.Name));
geomProps++;
}
}
if (geomProps > 1)
throw new FdoETLException("Source file cannot be copied to a SHP file. One or more feature classes have more than one geometry property");
}
}
}
}
}
else
{
if (!CreateFlatFileDataSource(targetPath))
throw new FdoException("Unable to create data source on: " + targetPath);
source = CreateFlatFileConnection(sourceFile);
target = CreateFlatFileConnection(targetPath);
}
//Source and target connections may have been opened before this point
if (source.State == FdoConnectionState.Closed)
source.Open();
if (target.State == FdoConnectionState.Closed)
target.Open();
string srcName = "SOURCE";
string dstName = "TARGET";
Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>();
connections.Add(srcName, source);
connections.Add(dstName, target);
options = new FdoBulkCopyOptions(connections, true);
if (copySpatialContexts)
{
CopyAllSpatialContexts(source, target, true);
}
using (FdoFeatureService srcService = source.CreateFeatureService())
using (FdoFeatureService destService = target.CreateFeatureService())
{
FeatureSchemaCollection schemas = srcService.DescribeSchema();
//Assume single-schema
FeatureSchema fs = schemas[0];
//Clone and apply to target
FeatureSchema targetSchema = FdoSchemaUtil.CloneSchema(fs);
IncompatibleSchema incSchema;
string sourceSchemaName = fs.Name;
string targetSchemaName = string.Empty;
bool canApply = destService.CanApplySchema(targetSchema, out incSchema);
if (canApply)
{
destService.ApplySchema(targetSchema);
targetSchemaName = targetSchema.Name;
}
else
{
if (fixIncompatibleSchema)
{
FeatureSchema fixedSchema = destService.AlterSchema(targetSchema, incSchema);
destService.ApplySchema(fixedSchema);
targetSchemaName = fixedSchema.Name;
}
else
{
throw new Exception(incSchema.ToString());
}
}
//Copy all classes
foreach (ClassDefinition cd in fs.Classes)
{
FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, sourceSchemaName, cd.Name, targetSchemaName, cd.Name);
copt.Name = "Copy source to target [" + cd.Name + "]";
copt.FlattenGeometries = flattenGeometries;
options.AddClassCopyOption(copt);
//Flick on batch support if we can
if (destService.SupportsBatchInsertion())
copt.BatchSize = 300; //Madness? THIS IS SPARTA!
}
}
}
catch (Exception)
{
if (source != null)
source.Dispose();
if (target != null)
target.Dispose();
throw;
}
return new FdoBulkCopy(options);
}
/// <summary>
/// Loads bulk copy options from deserialized xml
/// </summary>
/// <param name="def">The deserialized definition.</param>
/// <param name="name">The name.</param>
/// <param name="owner">if set to <c>true</c> [owner].</param>
/// <returns></returns>
public FdoBulkCopyOptions BulkCopyFromXml(FdoBulkCopyTaskDefinition def, ref string name, bool owner)
{
var nameMap = Prepare(def);
// TODO/FIXME/HACK:
//
// The introduction of on-the-fly schema modifications before copying has introduced a
// potential problem which will only occur when multiple copy tasks copy to the same target
// feature class.
//
// What happens is because a target will be multiple sources copying to it, if we need
// to create/update the target class definition, the current infrastructure will be confused
// as to which source feature class to clone from.
//
// A possible solution is to return the first matching name. This will cause a "create" operation
// to be queued. We then alter the pre-modification process so that subsequent "create" operations
// becomes "update" operations instead.
//
// On second thought, the current infrastructure should be smart enought to prevent this, as
// the UI only allows:
//
// 1. Mapping to an existing class
// 2. Mapping to a class of the same name (create if necessary)
//
// It won't be possible to map to a class that doesn't exist, as the requirement for auto-create/update
// is that the class name will be the same.
//
// ie. It won't be possible to create this mapping
//
// a. Foo -> Foo (create if necessary)
// b. Bar -> Foo
//
// Rule 1 prevents mapping b) from happening
// Rule 2 ensures that Foo will only ever be created once
//
// If multiple sources are mapped to the same target, I believe the bcp infrastructure should ensure
// that target must already exist first.
name = def.name;
Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>();
Dictionary<string, string> changeConnNames = new Dictionary<string, string>();
//TODO: Prepare() ensured that all connections have unique names that don't already exist
//now in the effort to re-use existing connections, see which connection entries
//already exist and can be renamed back to the old name
foreach (FdoConnectionEntryElement entry in def.Connections)
{
string connName = entry.name;
FdoConnection conn = CreateConnection(entry.provider, entry.ConnectionString, entry.configPath, ref connName);
connections[connName] = conn;
if (connName != entry.name)
{
changeConnNames[entry.name] = connName;
}
}
foreach (string oldName in changeConnNames.Keys)
{
def.UpdateConnectionReferences(oldName, changeConnNames[oldName]);
}
//Compile the list of classes to be queried
Dictionary<string, MultiSchemaQuery> queries = new Dictionary<string, MultiSchemaQuery>();
foreach (FdoCopyTaskElement task in def.CopyTasks)
{
MultiSchemaQuery src = null;
MultiSchemaQuery dst = null;
//Process source
if (!queries.ContainsKey(task.Source.connection))
src = queries[task.Source.connection] = new MultiSchemaQuery(task.Source.connection, SchemaOrigin.Source);
else
src = queries[task.Source.connection];
var sq = src.TryGet(task.Source.schema);
if (sq != null)
{
sq.AddClass(task.Source.@class);
}
else
{
sq = new SchemaQuery(task.Source.schema);
sq.AddClass(task.Source.@class);
src.Add(sq);
}
//Process target
if (!queries.ContainsKey(task.Target.connection))
dst = queries[task.Target.connection] = new MultiSchemaQuery(task.Target.connection, SchemaOrigin.Target);
else
dst = queries[task.Target.connection];
var tq = dst.TryGet(task.Target.schema);
if (tq != null)
{
tq.AddClass(task.Target.@class);
}
else
{
tq = new SchemaQuery(task.Target.schema);
tq.AddClass(task.Target.@class);
dst.Add(tq);
}
}
List<TargetClassModificationItem> modifiers = new List<TargetClassModificationItem>();
using (var schemaCache = new FeatureSchemaCache())
{
//Now populate the schema cache with source schemas
foreach (string connName in queries.Keys)
{
if (connections.ContainsKey(connName))
{
var mqry = queries[connName];
var conn = connections[connName];
FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);
using (var svc = conn.CreateFeatureService())
{
foreach (var sq in mqry.SchemaQueries)
{
//Source schema queries are expected to fully check out so use original method
if (mqry.Origin == SchemaOrigin.Source)
{
schemas.Add(svc.PartialDescribeSchema(sq.SchemaName, new List<string>(sq.ClassNames)));
}
}
}
if (schemas.Count > 0)
schemaCache.Add(connName, schemas);
}
}
//Now populate the schema cache with target schemas, taking note of any
//classes that need to be created or updated.
foreach (string connName in queries.Keys)
{
if (connections.ContainsKey(connName))
{
var mqry = queries[connName];
var conn = connections[connName];
FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);
using (var svc = conn.CreateFeatureService())
{
foreach (var sq in mqry.SchemaQueries)
{
//Source schema queries are expected to fully check out so use original method
if (mqry.Origin == SchemaOrigin.Target)
{
//Because we may need to create new classes, the old method will break down on non-existent class names
//So use the new method
string[] notFound;
var schema = svc.PartialDescribeSchema(sq.SchemaName, new List<string>(sq.ClassNames), out notFound);
//if (notFound.Length > 0)
//{
// //If we can't modify schemas we'll stop right here. This is caused by elements containing createIfNotExists = true
// if (!conn.Capability.GetBooleanCapability(CapabilityType.FdoCapabilityType_SupportsSchemaModification))
// throw new NotSupportedException("The connection named " + connName + " does not support schema modification. Therefore, copy tasks and property/expression mappings cannot have createIfNotExists = true");
// //This cumbersome, but we need the parent schema name of the class that we will need to copy
// string srcSchema = GetSourceSchemaForMapping(def, sq.SchemaName, notFound);
// foreach (string className in notFound)
// {
// modifiers.Add(new CreateTargetClassFromSource(srcSchema, className));
// }
//}
schemas.Add(schema);
}
}
}
if (schemas.Count > 0)
schemaCache.Add(connName, schemas);
}
}
FdoBulkCopyOptions opts = new FdoBulkCopyOptions(connections, owner);
foreach (FdoCopyTaskElement task in def.CopyTasks)
{
TargetClassModificationItem mod;
FdoClassCopyOptions copt = FdoClassCopyOptions.FromElement(task, schemaCache, connections[task.Source.connection], connections[task.Target.connection], out mod);
opts.AddClassCopyOption(copt);
if (mod != null)
copt.PreCopyTargetModifier = mod;
//opts.AddClassModifier(mod);
}
return opts;
}
}
/// <summary>
/// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
/// </summary>
/// <param name="options">The options.</param>
/// <param name="reportFrequency">The report frequency.</param>
public FdoBulkCopy(FdoBulkCopyOptions options, int reportFrequency)
: this(options)
{
_ReportFrequency = reportFrequency;
}
/// <summary>
/// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
/// </summary>
/// <param name="options">The options.</param>
public FdoBulkCopy(FdoBulkCopyOptions options)
{
_options = options;
}
