public AddClassCopyOption ( |
||
copt | The copt. | |
return | void |
public override int Execute() { CommandStatus retCode; FdoConnection srcConn = new FdoConnection(_srcProvider, _srcConnStr); FdoConnection destConn = null; //Directory given, assume SHP if (Directory.Exists(_destPath)) { destConn = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + _destPath); } else { if (ExpressUtility.CreateFlatFileDataSource(_destPath)) destConn = ExpressUtility.CreateFlatFileConnection(_destPath); else throw new FdoException("Could not create data source: " + _destPath); } try { srcConn.Open(); destConn.Open(); string srcName = "SOURCE"; string dstName = "TARGET"; FdoBulkCopyOptions options = new FdoBulkCopyOptions(); options.RegisterConnection(srcName, srcConn); options.RegisterConnection(dstName, destConn); using (FdoFeatureService srcService = srcConn.CreateFeatureService()) using (FdoFeatureService destService = destConn.CreateFeatureService()) { //See if spatial context needs to be copied to target if (!string.IsNullOrEmpty(_srcSpatialContext)) { SpatialContextInfo srcCtx = srcService.GetSpatialContext(_srcSpatialContext); if (srcCtx != null) { Console.WriteLine("Copying spatial context: " + srcCtx.Name); ExpressUtility.CopyAllSpatialContexts(new SpatialContextInfo[] { srcCtx }, destConn, true); } } else { //Copy all ExpressUtility.CopyAllSpatialContexts(srcConn, destConn, true); } FeatureSchema srcSchema = null; //See if partial class list is needed if (_srcClasses.Count > 0) { WriteLine("Checking if partial schema discovery is supported: " + srcService.SupportsPartialSchemaDiscovery()); srcSchema = srcService.PartialDescribeSchema(_srcSchema, _srcClasses); } else //Full copy { WriteLine("No classes specified, reading full source schema"); srcSchema = srcService.GetSchemaByName(_srcSchema); } if (srcSchema == null) { WriteError("Could not find source schema: " + _srcSchema); retCode = CommandStatus.E_FAIL_SCHEMA_NOT_FOUND; } else { WriteLine("Checking source schema for incompatibilities"); FeatureSchema targetSchema = null; IncompatibleSchema incSchema; if (destService.CanApplySchema(srcSchema, out incSchema)) { int clsCount = srcSchema.Classes.Count; WriteLine("Applying source schema (containing " + clsCount + " classes) to target"); destService.ApplySchema(srcSchema, null, true); targetSchema = srcSchema; } else { WriteWarning("Incompatibilities were detected in source schema. Applying a modified version to target"); FeatureSchema fixedSchema = destService.AlterSchema(srcSchema, incSchema); int clsCount = fixedSchema.Classes.Count; WriteLine("Applying modified source schema (containing " + clsCount + " classes) to target"); destService.ApplySchema(fixedSchema, null, true); targetSchema = fixedSchema; } //Now set class copy options foreach (ClassDefinition cd in srcSchema.Classes) { FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, srcSchema.Name, cd.Name, targetSchema.Name, cd.Name); copt.FlattenGeometries = _flatten; options.AddClassCopyOption(copt); } if (_flatten) { WriteWarning("The switch -flatten has been defined. Geometries that are copied will have any Z or M coordinates removed"); } FdoBulkCopy copy = new FdoBulkCopy(options); copy.ProcessMessage += new MessageEventHandler(OnMessage); copy.ProcessCompleted += new EventHandler(OnCompleted); Console.WriteLine("Executing bulk copy"); copy.Execute(); List<Exception> errors = new List<Exception>(copy.GetAllErrors()); if (errors.Count > 0) { string file = GenerateLogFileName("bcp-error-"); LogErrors(errors, file); base.WriteError("Errors were encountered during bulk copy."); retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS; } else { retCode = CommandStatus.E_OK; } retCode = CommandStatus.E_OK; } } } catch (Exception ex) { WriteException(ex); retCode = CommandStatus.E_FAIL_UNKNOWN; } finally { srcConn.Dispose(); destConn.Dispose(); } return (int)retCode; }
/// <summary> /// Creates a FDO bulk copy task. The target file will be created as part of /// this method call. If the target path is a directory, it is assumed that /// SHP files are to be created and copied to. /// </summary> /// <param name="sourceFile">The path to the source file.</param> /// <param name="targetPath"> /// The path to the target file/directory. If it is a directory, it is assumed /// that SHP files are to be created and copied to. /// </param> /// <param name="copySpatialContexts">If true, will also copy spatial contexts</param> /// <param name="fixIncompatibleSchema">If true, will try to fix the source schema to make it compatible with the target connection. If false, an exception will be thrown</param> /// <param name="flattenGeometries">If true, will strip all Z and M coordinates from all geometries that are copied</param> /// <returns></returns> public static FdoBulkCopy CreateBulkCopy(string sourceFile, string targetPath, bool copySpatialContexts, bool fixIncompatibleSchema, bool flattenGeometries) { FdoBulkCopyOptions options = null; FdoConnection source = null; FdoConnection target = null; try { //Is a directory. Implies a SHP connection if (IsShp(targetPath)) { //SHP doesn't actually support CreateDataStore. We use the following technique: // - Connect to base directory // - Clone source schema and apply to SHP connection. // - A SHP file and related files are created for each feature class. string shpdir = Directory.Exists(targetPath) ? targetPath : Path.GetDirectoryName(targetPath); source = CreateFlatFileConnection(sourceFile); target = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + shpdir); source.Open(); //Verify source has only classes with single geometry storage and only one geometry using (FdoFeatureService svc = source.CreateFeatureService()) { using (FeatureSchemaCollection schemas = svc.DescribeSchema()) { foreach (FeatureSchema sch in schemas) { foreach (ClassDefinition cd in sch.Classes) { int geomProps = 0; foreach (PropertyDefinition pd in cd.Properties) { if (pd.PropertyType == PropertyType.PropertyType_GeometricProperty) { GeometricPropertyDefinition gp = pd as GeometricPropertyDefinition; GeometricType[] types = FdoGeometryUtil.GetGeometricTypes(gp.GeometryTypes); if (types.Length != 1 || (types.Length == 1 && types[0] == GeometricType.GeometricType_All)) throw new FdoETLException(string.Format("Source file cannot be copied to a SHP file. {0}:{1}.{2} has more than one geometry storage type", sch.Name, cd.Name, pd.Name)); geomProps++; } } if (geomProps > 1) throw new FdoETLException("Source file cannot be copied to a SHP file. One or more feature classes have more than one geometry property"); } } } } } else { if (!CreateFlatFileDataSource(targetPath)) throw new FdoException("Unable to create data source on: " + targetPath); source = CreateFlatFileConnection(sourceFile); target = CreateFlatFileConnection(targetPath); } //Source and target connections may have been opened before this point if (source.State == FdoConnectionState.Closed) source.Open(); if (target.State == FdoConnectionState.Closed) target.Open(); string srcName = "SOURCE"; string dstName = "TARGET"; Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>(); connections.Add(srcName, source); connections.Add(dstName, target); options = new FdoBulkCopyOptions(connections, true); if (copySpatialContexts) { CopyAllSpatialContexts(source, target, true); } using (FdoFeatureService srcService = source.CreateFeatureService()) using (FdoFeatureService destService = target.CreateFeatureService()) { FeatureSchemaCollection schemas = srcService.DescribeSchema(); //Assume single-schema FeatureSchema fs = schemas[0]; //Clone and apply to target FeatureSchema targetSchema = FdoSchemaUtil.CloneSchema(fs); IncompatibleSchema incSchema; string sourceSchemaName = fs.Name; string targetSchemaName = string.Empty; bool canApply = destService.CanApplySchema(targetSchema, out incSchema); if (canApply) { destService.ApplySchema(targetSchema); targetSchemaName = targetSchema.Name; } else { if (fixIncompatibleSchema) { FeatureSchema fixedSchema = destService.AlterSchema(targetSchema, incSchema); destService.ApplySchema(fixedSchema); targetSchemaName = fixedSchema.Name; } else { throw new Exception(incSchema.ToString()); } } //Copy all classes foreach (ClassDefinition cd in fs.Classes) { FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, sourceSchemaName, cd.Name, targetSchemaName, cd.Name); copt.Name = "Copy source to target [" + cd.Name + "]"; copt.FlattenGeometries = flattenGeometries; options.AddClassCopyOption(copt); //Flick on batch support if we can if (destService.SupportsBatchInsertion()) copt.BatchSize = 300; //Madness? THIS IS SPARTA! } } } catch (Exception) { if (source != null) source.Dispose(); if (target != null) target.Dispose(); throw; } return new FdoBulkCopy(options); }
/// <summary> /// Utility method to create a bulk copy operation from /// one class to another /// </summary> /// <param name="sourceConn"></param> /// <param name="targetConn"></param> /// <param name="srcSchemaName"></param> /// <param name="srcQuery"></param> /// <param name="targetSchemaName"></param> /// <param name="targetClassName"></param> /// <param name="propertyMapping"></param> /// <returns></returns> public static FdoBulkCopy CreateBulkCopy( FdoConnection sourceConn, FdoConnection targetConn, string srcSchemaName, FeatureQueryOptions srcQuery, string targetSchemaName, string targetClassName, NameValueCollection propertyMapping) { var dict = new Dictionary<string, FdoConnection>(); dict["SOURCE"] = sourceConn; dict["TARGET"] = targetConn; var opts = new FdoBulkCopyOptions(dict, false); var copt = new FdoClassCopyOptions( "SOURCE", "TARGET", srcSchemaName, srcQuery.ClassName, targetSchemaName, targetClassName); if (!string.IsNullOrEmpty(srcQuery.Filter)) copt.SourceFilter = srcQuery.Filter; foreach (string p in propertyMapping.Keys) { copt.AddPropertyMapping(p, propertyMapping[p]); } copt.FlattenGeometries = true; copt.ForceWkb = true; opts.AddClassCopyOption(copt); return new FdoBulkCopy(opts, 100); }
/// <summary> /// Loads bulk copy options from deserialized xml /// </summary> /// <param name="def">The deserialized definition.</param> /// <param name="name">The name.</param> /// <param name="owner">if set to <c>true</c> [owner].</param> /// <returns></returns> public FdoBulkCopyOptions BulkCopyFromXml(FdoBulkCopyTaskDefinition def, ref string name, bool owner) { var nameMap = Prepare(def); // TODO/FIXME/HACK: // // The introduction of on-the-fly schema modifications before copying has introduced a // potential problem which will only occur when multiple copy tasks copy to the same target // feature class. // // What happens is because a target will be multiple sources copying to it, if we need // to create/update the target class definition, the current infrastructure will be confused // as to which source feature class to clone from. // // A possible solution is to return the first matching name. This will cause a "create" operation // to be queued. We then alter the pre-modification process so that subsequent "create" operations // becomes "update" operations instead. // // On second thought, the current infrastructure should be smart enought to prevent this, as // the UI only allows: // // 1. Mapping to an existing class // 2. Mapping to a class of the same name (create if necessary) // // It won't be possible to map to a class that doesn't exist, as the requirement for auto-create/update // is that the class name will be the same. // // ie. It won't be possible to create this mapping // // a. Foo -> Foo (create if necessary) // b. Bar -> Foo // // Rule 1 prevents mapping b) from happening // Rule 2 ensures that Foo will only ever be created once // // If multiple sources are mapped to the same target, I believe the bcp infrastructure should ensure // that target must already exist first. name = def.name; Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>(); Dictionary<string, string> changeConnNames = new Dictionary<string, string>(); //TODO: Prepare() ensured that all connections have unique names that don't already exist //now in the effort to re-use existing connections, see which connection entries //already exist and can be renamed back to the old name foreach (FdoConnectionEntryElement entry in def.Connections) { string connName = entry.name; FdoConnection conn = CreateConnection(entry.provider, entry.ConnectionString, entry.configPath, ref connName); connections[connName] = conn; if (connName != entry.name) { changeConnNames[entry.name] = connName; } } foreach (string oldName in changeConnNames.Keys) { def.UpdateConnectionReferences(oldName, changeConnNames[oldName]); } //Compile the list of classes to be queried Dictionary<string, MultiSchemaQuery> queries = new Dictionary<string, MultiSchemaQuery>(); foreach (FdoCopyTaskElement task in def.CopyTasks) { MultiSchemaQuery src = null; MultiSchemaQuery dst = null; //Process source if (!queries.ContainsKey(task.Source.connection)) src = queries[task.Source.connection] = new MultiSchemaQuery(task.Source.connection, SchemaOrigin.Source); else src = queries[task.Source.connection]; var sq = src.TryGet(task.Source.schema); if (sq != null) { sq.AddClass(task.Source.@class); } else { sq = new SchemaQuery(task.Source.schema); sq.AddClass(task.Source.@class); src.Add(sq); } //Process target if (!queries.ContainsKey(task.Target.connection)) dst = queries[task.Target.connection] = new MultiSchemaQuery(task.Target.connection, SchemaOrigin.Target); else dst = queries[task.Target.connection]; var tq = dst.TryGet(task.Target.schema); if (tq != null) { tq.AddClass(task.Target.@class); } else { tq = new SchemaQuery(task.Target.schema); tq.AddClass(task.Target.@class); dst.Add(tq); } } List<TargetClassModificationItem> modifiers = new List<TargetClassModificationItem>(); using (var schemaCache = new FeatureSchemaCache()) { //Now populate the schema cache with source schemas foreach (string connName in queries.Keys) { if (connections.ContainsKey(connName)) { var mqry = queries[connName]; var conn = connections[connName]; FeatureSchemaCollection schemas = new FeatureSchemaCollection(null); using (var svc = conn.CreateFeatureService()) { foreach (var sq in mqry.SchemaQueries) { //Source schema queries are expected to fully check out so use original method if (mqry.Origin == SchemaOrigin.Source) { schemas.Add(svc.PartialDescribeSchema(sq.SchemaName, new List<string>(sq.ClassNames))); } } } if (schemas.Count > 0) schemaCache.Add(connName, schemas); } } //Now populate the schema cache with target schemas, taking note of any //classes that need to be created or updated. foreach (string connName in queries.Keys) { if (connections.ContainsKey(connName)) { var mqry = queries[connName]; var conn = connections[connName]; FeatureSchemaCollection schemas = new FeatureSchemaCollection(null); using (var svc = conn.CreateFeatureService()) { foreach (var sq in mqry.SchemaQueries) { //Source schema queries are expected to fully check out so use original method if (mqry.Origin == SchemaOrigin.Target) { //Because we may need to create new classes, the old method will break down on non-existent class names //So use the new method string[] notFound; var schema = svc.PartialDescribeSchema(sq.SchemaName, new List<string>(sq.ClassNames), out notFound); //if (notFound.Length > 0) //{ // //If we can't modify schemas we'll stop right here. This is caused by elements containing createIfNotExists = true // if (!conn.Capability.GetBooleanCapability(CapabilityType.FdoCapabilityType_SupportsSchemaModification)) // throw new NotSupportedException("The connection named " + connName + " does not support schema modification. Therefore, copy tasks and property/expression mappings cannot have createIfNotExists = true"); // //This cumbersome, but we need the parent schema name of the class that we will need to copy // string srcSchema = GetSourceSchemaForMapping(def, sq.SchemaName, notFound); // foreach (string className in notFound) // { // modifiers.Add(new CreateTargetClassFromSource(srcSchema, className)); // } //} schemas.Add(schema); } } } if (schemas.Count > 0) schemaCache.Add(connName, schemas); } } FdoBulkCopyOptions opts = new FdoBulkCopyOptions(connections, owner); foreach (FdoCopyTaskElement task in def.CopyTasks) { TargetClassModificationItem mod; FdoClassCopyOptions copt = FdoClassCopyOptions.FromElement(task, schemaCache, connections[task.Source.connection], connections[task.Target.connection], out mod); opts.AddClassCopyOption(copt); if (mod != null) copt.PreCopyTargetModifier = mod; //opts.AddClassModifier(mod); } return opts; } }