public PreClassCopyModifyOperation(FdoClassCopyOptions opts, FdoConnection source, FdoConnection target) { if (opts.PreCopyTargetModifier == null) { throw new ArgumentException("No pre-copy modifier specified"); } _source = source; _target = target; _opts = opts; }
/// <summary> /// Adds the class copy option. /// </summary> /// <param name="copt">The copt.</param> public void AddClassCopyOption(FdoClassCopyOptions copt) { copt.Parent = this; _copyOptions.Add(copt); }
internal static FdoClassCopyOptions FromElement(FdoCopyTaskElement el, FeatureSchemaCache cache, FdoConnection sourceConn, FdoConnection targetConn, out TargetClassModificationItem mod) { mod = null; if (!cache.HasConnection(el.Source.connection)) throw new TaskLoaderException("The referenced source connection is not defined"); if (!cache.HasConnection(el.Target.connection)) throw new TaskLoaderException("The referenced target connection is not defined"); FdoClassCopyOptions opts = new FdoClassCopyOptions(el.Source.connection, el.Target.connection, el.Source.schema, el.Source.@class, el.Target.schema, el.Target.@class); opts.DeleteTarget = el.Options.DeleteTarget; opts.SourceFilter = el.Options.Filter; if (!el.Options.FlattenGeometriesSpecified) opts.FlattenGeometries = false; else opts.FlattenGeometries = el.Options.FlattenGeometries; if (!el.Options.ForceWKBSpecified) opts.ForceWkb = false; else opts.ForceWkb = el.Options.ForceWKB; if (!string.IsNullOrEmpty(el.Options.BatchSize)) opts.BatchSize = Convert.ToInt32(el.Options.BatchSize); opts.Name = el.name; opts.CreateIfNotExists = el.createIfNotExists; ClassDefinition srcClass = cache.GetClassByName(el.Source.connection, el.Source.schema, el.Source.@class); ClassDefinition dstClass = cache.GetClassByName(el.Target.connection, el.Target.schema, el.Target.@class); if (!el.createIfNotExists && dstClass == null) throw new InvalidOperationException("Target class " + el.Target.@class + " does not exist and the createIfNotExist option is false"); SpatialContextInfo defaultSc = null; FunctionDefinitionCollection availableFunctions = (FunctionDefinitionCollection)sourceConn.Capability.GetObjectCapability(CapabilityType.FdoCapabilityType_ExpressionFunctions); using (var svc = targetConn.CreateFeatureService()) { defaultSc = svc.GetActiveSpatialContext(); } if (dstClass != null) { foreach (FdoPropertyMappingElement propMap in el.PropertyMappings) { if (srcClass.Properties.IndexOf(propMap.source) < 0) throw new TaskLoaderException("The property mapping (" + propMap.source + " -> " + propMap.target + ") in task (" + el.name + ") contains a source property not found in the source class definition (" + el.Source.@class + ")"); //Add to list of properties to check for if (propMap.createIfNotExists && dstClass.Properties.IndexOf(propMap.target) < 0) { if (mod == null) mod = new UpdateTargetClass(dstClass.Name); opts.AddSourcePropertyToCheck(propMap.source); //Clone copy of source property of same name var srcProp = srcClass.Properties[srcClass.Properties.IndexOf(propMap.source)]; srcProp = FdoSchemaUtil.CloneProperty(srcProp); mod.AddProperty(srcProp); } else { if (dstClass.Properties.IndexOf(propMap.target) < 0) throw new TaskLoaderException("The property mapping (" + propMap.source + " -> " + propMap.target + ") in task (" + el.name + ") contains a target property not found in the target class definition (" + el.Target.@class + ")"); PropertyDefinition sp = srcClass.Properties[propMap.source]; PropertyDefinition tp = dstClass.Properties[propMap.target]; if (sp.PropertyType != tp.PropertyType) throw new TaskLoaderException("The properties in the mapping (" + propMap.source + " -> " + propMap.target + ") are of different types"); //if (sp.PropertyType != PropertyType.PropertyType_DataProperty) // throw new TaskLoaderException("One or more properties in the mapping (" + propMap.source + " -> " + propMap.target + ") is not a data property"); DataPropertyDefinition sdp = sp as DataPropertyDefinition; DataPropertyDefinition tdp = tp as DataPropertyDefinition; opts.AddPropertyMapping(propMap.source, propMap.target); //Property mapping is between two data properties if (sdp != null && tdp != null) { //Types not equal, so add a conversion rule if (sdp.DataType != tdp.DataType) { FdoDataPropertyConversionRule rule = new FdoDataPropertyConversionRule( propMap.source, propMap.target, sdp.DataType, tdp.DataType, propMap.nullOnFailedConversion, propMap.truncate); opts.AddDataConversionRule(propMap.source, rule); } } } } // foreach (FdoExpressionMappingElement exprMap in el.ExpressionMappings) { if (string.IsNullOrEmpty(exprMap.target)) continue; opts.AddSourceExpression(exprMap.alias, exprMap.Expression, exprMap.target); //Add to list of properties to check for if (exprMap.createIfNotExists) { //Class exists but property doesn't if (dstClass.Properties.IndexOf(exprMap.target) < 0) { if (mod == null) { mod = new UpdateTargetClass(el.Target.@class); } var prop = FdoSchemaUtil.CreatePropertyFromExpressionType(exprMap.Expression, srcClass, availableFunctions, defaultSc.Name); if (prop == null) { throw new InvalidOperationException("Could not derive a property definition from the expression: " + exprMap.Expression); } prop.Name = exprMap.target; mod.AddProperty(prop); } } else //Conversion rules can only apply if both properties exist. { FdoPropertyType? pt = ExpressionUtility.ParseExpressionType(exprMap.Expression, sourceConn); if (pt.HasValue) { DataType? srcDt = ValueConverter.GetDataType(pt.Value); if (srcDt.HasValue) { PropertyDefinition tp = dstClass.Properties[exprMap.target]; DataPropertyDefinition tdp = tp as DataPropertyDefinition; if (tdp != null) { if (srcDt.Value != tdp.DataType) { FdoDataPropertyConversionRule rule = new FdoDataPropertyConversionRule( exprMap.alias, exprMap.target, srcDt.Value, tdp.DataType, exprMap.nullOnFailedConversion, exprMap.truncate); opts.AddDataConversionRule(exprMap.alias, rule); } } } } } } } else //class doesn't exist { mod = new CreateTargetClassFromSource(el.Source.schema, el.Target.@class); foreach (var propMap in el.PropertyMappings) { opts.AddPropertyMapping(propMap.source, propMap.target); if (propMap.createIfNotExists) { opts.AddSourcePropertyToCheck(propMap.source); } } foreach (var exprMap in el.ExpressionMappings) { opts.AddSourceExpression(exprMap.alias, exprMap.Expression, exprMap.target); if (exprMap.createIfNotExists) opts.AddSourcePropertyToCheck(exprMap.alias); var prop = FdoSchemaUtil.CreatePropertyFromExpressionType(exprMap.Expression, srcClass, availableFunctions, defaultSc.Name); if (prop == null) { throw new InvalidOperationException("Could not derive a property definition from the expression: " + exprMap.Expression); } prop.Name = exprMap.target; mod.AddProperty(prop); } } return opts; }
/// <summary> /// Initializes a new instance of the <see cref="FdoClassToClassCopyProcess"/> class. /// </summary> /// <param name="options">The options.</param> public FdoClassToClassCopyProcess(FdoClassCopyOptions options) { this.Options = options; }
public override int Execute() { CommandStatus retCode; FdoConnection srcConn = new FdoConnection(_srcProvider, _srcConnStr); FdoConnection destConn = null; //Directory given, assume SHP if (Directory.Exists(_destPath)) { destConn = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + _destPath); } else { if (ExpressUtility.CreateFlatFileDataSource(_destPath)) destConn = ExpressUtility.CreateFlatFileConnection(_destPath); else throw new FdoException("Could not create data source: " + _destPath); } try { srcConn.Open(); destConn.Open(); string srcName = "SOURCE"; string dstName = "TARGET"; FdoBulkCopyOptions options = new FdoBulkCopyOptions(); options.RegisterConnection(srcName, srcConn); options.RegisterConnection(dstName, destConn); using (FdoFeatureService srcService = srcConn.CreateFeatureService()) using (FdoFeatureService destService = destConn.CreateFeatureService()) { //See if spatial context needs to be copied to target if (!string.IsNullOrEmpty(_srcSpatialContext)) { SpatialContextInfo srcCtx = srcService.GetSpatialContext(_srcSpatialContext); if (srcCtx != null) { Console.WriteLine("Copying spatial context: " + srcCtx.Name); ExpressUtility.CopyAllSpatialContexts(new SpatialContextInfo[] { srcCtx }, destConn, true); } } else { //Copy all ExpressUtility.CopyAllSpatialContexts(srcConn, destConn, true); } FeatureSchema srcSchema = null; //See if partial class list is needed if (_srcClasses.Count > 0) { WriteLine("Checking if partial schema discovery is supported: " + srcService.SupportsPartialSchemaDiscovery()); srcSchema = srcService.PartialDescribeSchema(_srcSchema, _srcClasses); } else //Full copy { WriteLine("No classes specified, reading full source schema"); srcSchema = srcService.GetSchemaByName(_srcSchema); } if (srcSchema == null) { WriteError("Could not find source schema: " + _srcSchema); retCode = CommandStatus.E_FAIL_SCHEMA_NOT_FOUND; } else { WriteLine("Checking source schema for incompatibilities"); FeatureSchema targetSchema = null; IncompatibleSchema incSchema; if (destService.CanApplySchema(srcSchema, out incSchema)) { int clsCount = srcSchema.Classes.Count; WriteLine("Applying source schema (containing " + clsCount + " classes) to target"); destService.ApplySchema(srcSchema, null, true); targetSchema = srcSchema; } else { WriteWarning("Incompatibilities were detected in source schema. Applying a modified version to target"); FeatureSchema fixedSchema = destService.AlterSchema(srcSchema, incSchema); int clsCount = fixedSchema.Classes.Count; WriteLine("Applying modified source schema (containing " + clsCount + " classes) to target"); destService.ApplySchema(fixedSchema, null, true); targetSchema = fixedSchema; } //Now set class copy options foreach (ClassDefinition cd in srcSchema.Classes) { FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, srcSchema.Name, cd.Name, targetSchema.Name, cd.Name); copt.FlattenGeometries = _flatten; options.AddClassCopyOption(copt); } if (_flatten) { WriteWarning("The switch -flatten has been defined. Geometries that are copied will have any Z or M coordinates removed"); } FdoBulkCopy copy = new FdoBulkCopy(options); copy.ProcessMessage += new MessageEventHandler(OnMessage); copy.ProcessCompleted += new EventHandler(OnCompleted); Console.WriteLine("Executing bulk copy"); copy.Execute(); List<Exception> errors = new List<Exception>(copy.GetAllErrors()); if (errors.Count > 0) { string file = GenerateLogFileName("bcp-error-"); LogErrors(errors, file); base.WriteError("Errors were encountered during bulk copy."); retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS; } else { retCode = CommandStatus.E_OK; } retCode = CommandStatus.E_OK; } } } catch (Exception ex) { WriteException(ex); retCode = CommandStatus.E_FAIL_UNKNOWN; } finally { srcConn.Dispose(); destConn.Dispose(); } return (int)retCode; }
internal static FdoClassCopyOptions FromElement(FdoCopyTaskElement el, FeatureSchemaCache cache, FdoConnection sourceConn, FdoConnection targetConn, out TargetClassModificationItem mod) { mod = null; if (!cache.HasConnection(el.Source.connection)) { throw new TaskLoaderException("The referenced source connection is not defined"); } if (!cache.HasConnection(el.Target.connection)) { throw new TaskLoaderException("The referenced target connection is not defined"); } FdoClassCopyOptions opts = new FdoClassCopyOptions(el.Source.connection, el.Target.connection, el.Source.schema, el.Source.@class, el.Target.schema, el.Target.@class); opts.DeleteTarget = el.Options.DeleteTarget; opts.SourceFilter = el.Options.Filter; if (!el.Options.FlattenGeometriesSpecified) { opts.FlattenGeometries = false; } else { opts.FlattenGeometries = el.Options.FlattenGeometries; } if (!el.Options.ForceWKBSpecified) { opts.ForceWkb = false; } else { opts.ForceWkb = el.Options.ForceWKB; } if (!string.IsNullOrEmpty(el.Options.BatchSize)) { opts.BatchSize = Convert.ToInt32(el.Options.BatchSize); } opts.Name = el.name; opts.CreateIfNotExists = el.createIfNotExists; ClassDefinition srcClass = cache.GetClassByName(el.Source.connection, el.Source.schema, el.Source.@class); ClassDefinition dstClass = cache.GetClassByName(el.Target.connection, el.Target.schema, el.Target.@class); if (!el.createIfNotExists && dstClass == null) { throw new InvalidOperationException("Target class " + el.Target.@class + " does not exist and the createIfNotExist option is false"); } SpatialContextInfo defaultSc = null; FunctionDefinitionCollection availableFunctions = (FunctionDefinitionCollection)sourceConn.Capability.GetObjectCapability(CapabilityType.FdoCapabilityType_ExpressionFunctions); using (var svc = targetConn.CreateFeatureService()) { defaultSc = svc.GetActiveSpatialContext(); } if (dstClass != null) { foreach (FdoPropertyMappingElement propMap in el.PropertyMappings) { if (srcClass.Properties.IndexOf(propMap.source) < 0) { throw new TaskLoaderException("The property mapping (" + propMap.source + " -> " + propMap.target + ") in task (" + el.name + ") contains a source property not found in the source class definition (" + el.Source.@class + ")"); } //Add to list of properties to check for if (propMap.createIfNotExists && dstClass.Properties.IndexOf(propMap.target) < 0) { if (mod == null) { mod = new UpdateTargetClass(dstClass.Name); } opts.AddSourcePropertyToCheck(propMap.source); //Clone copy of source property of same name var srcProp = srcClass.Properties[srcClass.Properties.IndexOf(propMap.source)]; srcProp = FdoSchemaUtil.CloneProperty(srcProp); mod.AddProperty(srcProp); } else { if (dstClass.Properties.IndexOf(propMap.target) < 0) { throw new TaskLoaderException("The property mapping (" + propMap.source + " -> " + propMap.target + ") in task (" + el.name + ") contains a target property not found in the target class definition (" + el.Target.@class + ")"); } PropertyDefinition sp = srcClass.Properties[propMap.source]; PropertyDefinition tp = dstClass.Properties[propMap.target]; if (sp.PropertyType != tp.PropertyType) { throw new TaskLoaderException("The properties in the mapping (" + propMap.source + " -> " + propMap.target + ") are of different types"); } //if (sp.PropertyType != PropertyType.PropertyType_DataProperty) // throw new TaskLoaderException("One or more properties in the mapping (" + propMap.source + " -> " + propMap.target + ") is not a data property"); DataPropertyDefinition sdp = sp as DataPropertyDefinition; DataPropertyDefinition tdp = tp as DataPropertyDefinition; opts.AddPropertyMapping(propMap.source, propMap.target); //Property mapping is between two data properties if (sdp != null && tdp != null) { //Types not equal, so add a conversion rule if (sdp.DataType != tdp.DataType) { FdoDataPropertyConversionRule rule = new FdoDataPropertyConversionRule( propMap.source, propMap.target, sdp.DataType, tdp.DataType, propMap.nullOnFailedConversion, propMap.truncate); opts.AddDataConversionRule(propMap.source, rule); } } } } // foreach (FdoExpressionMappingElement exprMap in el.ExpressionMappings) { if (string.IsNullOrEmpty(exprMap.target)) { continue; } opts.AddSourceExpression(exprMap.alias, exprMap.Expression, exprMap.target); //Add to list of properties to check for if (exprMap.createIfNotExists) { //Class exists but property doesn't if (dstClass.Properties.IndexOf(exprMap.target) < 0) { if (mod == null) { mod = new UpdateTargetClass(el.Target.@class); } var prop = FdoSchemaUtil.CreatePropertyFromExpressionType(exprMap.Expression, srcClass, availableFunctions, defaultSc.Name); if (prop == null) { throw new InvalidOperationException("Could not derive a property definition from the expression: " + exprMap.Expression); } prop.Name = exprMap.target; mod.AddProperty(prop); } } else //Conversion rules can only apply if both properties exist. { FdoPropertyType?pt = ExpressionUtility.ParseExpressionType(exprMap.Expression, sourceConn); if (pt.HasValue) { DataType?srcDt = ValueConverter.GetDataType(pt.Value); if (srcDt.HasValue) { PropertyDefinition tp = dstClass.Properties[exprMap.target]; DataPropertyDefinition tdp = tp as DataPropertyDefinition; if (tdp != null) { if (srcDt.Value != tdp.DataType) { FdoDataPropertyConversionRule rule = new FdoDataPropertyConversionRule( exprMap.alias, exprMap.target, srcDt.Value, tdp.DataType, exprMap.nullOnFailedConversion, exprMap.truncate); opts.AddDataConversionRule(exprMap.alias, rule); } } } } } } } else //class doesn't exist { mod = new CreateTargetClassFromSource(el.Source.schema, el.Target.@class); foreach (var propMap in el.PropertyMappings) { opts.AddPropertyMapping(propMap.source, propMap.target); if (propMap.createIfNotExists) { opts.AddSourcePropertyToCheck(propMap.source); } } foreach (var exprMap in el.ExpressionMappings) { opts.AddSourceExpression(exprMap.alias, exprMap.Expression, exprMap.target); if (exprMap.createIfNotExists) { opts.AddSourcePropertyToCheck(exprMap.alias); } var prop = FdoSchemaUtil.CreatePropertyFromExpressionType(exprMap.Expression, srcClass, availableFunctions, defaultSc.Name); if (prop == null) { throw new InvalidOperationException("Could not derive a property definition from the expression: " + exprMap.Expression); } prop.Name = exprMap.target; mod.AddProperty(prop); } } return(opts); }
/// <summary> /// Utility method to create a bulk copy operation from /// one class to another /// </summary> /// <param name="sourceConn"></param> /// <param name="targetConn"></param> /// <param name="srcSchemaName"></param> /// <param name="srcQuery"></param> /// <param name="targetSchemaName"></param> /// <param name="targetClassName"></param> /// <param name="propertyMapping"></param> /// <returns></returns> public static FdoBulkCopy CreateBulkCopy( FdoConnection sourceConn, FdoConnection targetConn, string srcSchemaName, FeatureQueryOptions srcQuery, string targetSchemaName, string targetClassName, NameValueCollection propertyMapping) { var dict = new Dictionary<string, FdoConnection>(); dict["SOURCE"] = sourceConn; dict["TARGET"] = targetConn; var opts = new FdoBulkCopyOptions(dict, false); var copt = new FdoClassCopyOptions( "SOURCE", "TARGET", srcSchemaName, srcQuery.ClassName, targetSchemaName, targetClassName); if (!string.IsNullOrEmpty(srcQuery.Filter)) copt.SourceFilter = srcQuery.Filter; foreach (string p in propertyMapping.Keys) { copt.AddPropertyMapping(p, propertyMapping[p]); } copt.FlattenGeometries = true; copt.ForceWkb = true; opts.AddClassCopyOption(copt); return new FdoBulkCopy(opts, 100); }
/// <summary> /// Creates a FDO bulk copy task. The target file will be created as part of /// this method call. If the target path is a directory, it is assumed that /// SHP files are to be created and copied to. /// </summary> /// <param name="sourceFile">The path to the source file.</param> /// <param name="targetPath"> /// The path to the target file/directory. If it is a directory, it is assumed /// that SHP files are to be created and copied to. /// </param> /// <param name="copySpatialContexts">If true, will also copy spatial contexts</param> /// <param name="fixIncompatibleSchema">If true, will try to fix the source schema to make it compatible with the target connection. If false, an exception will be thrown</param> /// <param name="flattenGeometries">If true, will strip all Z and M coordinates from all geometries that are copied</param> /// <returns></returns> public static FdoBulkCopy CreateBulkCopy(string sourceFile, string targetPath, bool copySpatialContexts, bool fixIncompatibleSchema, bool flattenGeometries) { FdoBulkCopyOptions options = null; FdoConnection source = null; FdoConnection target = null; try { //Is a directory. Implies a SHP connection if (IsShp(targetPath)) { //SHP doesn't actually support CreateDataStore. We use the following technique: // - Connect to base directory // - Clone source schema and apply to SHP connection. // - A SHP file and related files are created for each feature class. string shpdir = Directory.Exists(targetPath) ? targetPath : Path.GetDirectoryName(targetPath); source = CreateFlatFileConnection(sourceFile); target = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + shpdir); source.Open(); //Verify source has only classes with single geometry storage and only one geometry using (FdoFeatureService svc = source.CreateFeatureService()) { using (FeatureSchemaCollection schemas = svc.DescribeSchema()) { foreach (FeatureSchema sch in schemas) { foreach (ClassDefinition cd in sch.Classes) { int geomProps = 0; foreach (PropertyDefinition pd in cd.Properties) { if (pd.PropertyType == PropertyType.PropertyType_GeometricProperty) { GeometricPropertyDefinition gp = pd as GeometricPropertyDefinition; GeometricType[] types = FdoGeometryUtil.GetGeometricTypes(gp.GeometryTypes); if (types.Length != 1 || (types.Length == 1 && types[0] == GeometricType.GeometricType_All)) throw new FdoETLException(string.Format("Source file cannot be copied to a SHP file. {0}:{1}.{2} has more than one geometry storage type", sch.Name, cd.Name, pd.Name)); geomProps++; } } if (geomProps > 1) throw new FdoETLException("Source file cannot be copied to a SHP file. One or more feature classes have more than one geometry property"); } } } } } else { if (!CreateFlatFileDataSource(targetPath)) throw new FdoException("Unable to create data source on: " + targetPath); source = CreateFlatFileConnection(sourceFile); target = CreateFlatFileConnection(targetPath); } //Source and target connections may have been opened before this point if (source.State == FdoConnectionState.Closed) source.Open(); if (target.State == FdoConnectionState.Closed) target.Open(); string srcName = "SOURCE"; string dstName = "TARGET"; Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>(); connections.Add(srcName, source); connections.Add(dstName, target); options = new FdoBulkCopyOptions(connections, true); if (copySpatialContexts) { CopyAllSpatialContexts(source, target, true); } using (FdoFeatureService srcService = source.CreateFeatureService()) using (FdoFeatureService destService = target.CreateFeatureService()) { FeatureSchemaCollection schemas = srcService.DescribeSchema(); //Assume single-schema FeatureSchema fs = schemas[0]; //Clone and apply to target FeatureSchema targetSchema = FdoSchemaUtil.CloneSchema(fs); IncompatibleSchema incSchema; string sourceSchemaName = fs.Name; string targetSchemaName = string.Empty; bool canApply = destService.CanApplySchema(targetSchema, out incSchema); if (canApply) { destService.ApplySchema(targetSchema); targetSchemaName = targetSchema.Name; } else { if (fixIncompatibleSchema) { FeatureSchema fixedSchema = destService.AlterSchema(targetSchema, incSchema); destService.ApplySchema(fixedSchema); targetSchemaName = fixedSchema.Name; } else { throw new Exception(incSchema.ToString()); } } //Copy all classes foreach (ClassDefinition cd in fs.Classes) { FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, sourceSchemaName, cd.Name, targetSchemaName, cd.Name); copt.Name = "Copy source to target [" + cd.Name + "]"; copt.FlattenGeometries = flattenGeometries; options.AddClassCopyOption(copt); //Flick on batch support if we can if (destService.SupportsBatchInsertion()) copt.BatchSize = 300; //Madness? THIS IS SPARTA! } } } catch (Exception) { if (source != null) source.Dispose(); if (target != null) target.Dispose(); throw; } return new FdoBulkCopy(options); }
public PreClassCopyModifyOperation(FdoClassCopyOptions opts, FdoConnection source, FdoConnection target) { if (opts.PreCopyTargetModifier == null) throw new ArgumentException("No pre-copy modifier specified"); _source = source; _target = target; _opts = opts; }