Defines the options for a FdoBulkCopy instance
Inheritance: IDisposable
        private void Load(FdoBulkCopyOptions def, string name)
        {
            txtName.Text = name;

            grdConnections.Rows.Clear();

            foreach (string connName in def.ConnectionNames)
            {
                this.AddParticipatingConnection(connName);
            }

            TreeNode root = mTreeView.Nodes[0];
            foreach (FdoClassCopyOptions task in def.ClassCopyOptions)
            {
                //Init w/ defaults
                CopyTaskNodeDecorator dec = AddNewTask(
                                               root,
                                               task.SourceConnectionName,
                                               task.SourceSchema,
                                               task.SourceClassName,
                                               task.TargetConnectionName,
                                               task.TargetSchema,
                                               task.TargetClassName,
                                               task.Name,
                                               task.CreateIfNotExists);

                _tasks[dec.DecoratedNode.Index] = dec;
                root.Expand();

                btnSave.Enabled = (root.Nodes.Count > 0);

                //Options
                dec.Options.BatchSize = task.BatchSize;
                dec.Options.Delete = task.DeleteTarget;
                dec.Options.SourceFilter = task.SourceFilter;
                dec.Options.Flatten = task.FlattenGeometries;

                var checkProps = new List<string>(task.CheckSourceProperties);
                //Property Mappings
                foreach (string srcProp in task.SourcePropertyNames)
                {
                    string dstProp = task.GetTargetProperty(srcProp);
                    bool createIfNotExists = checkProps.Contains(srcProp);
                    try
                    {
                        dec.PropertyMappings.MapProperty(srcProp, dstProp, createIfNotExists);
                    }
                    catch (MappingException ex)
                    {
                        LoggingService.Info("Skipping mapping: " + srcProp + " => " + dstProp + " (" + ex.Message + ")");
                    }
                    FdoDataPropertyConversionRule rule = task.GetDataConversionRule(srcProp);
                    PropertyConversionNodeDecorator cd = dec.PropertyMappings.GetConversionRule(srcProp);
                    if (rule != null)
                    {
                        cd.NullOnFailedConversion = rule.NullOnFailure;
                        cd.Truncate = rule.Truncate;
                    }
                }

                //Expression Mappings
                foreach (string alias in task.SourceAliases)
                {
                    string expr = task.GetExpression(alias);
                    string dstProp = task.GetTargetPropertyForAlias(alias);
                    bool createIfNotExists = checkProps.Contains(alias);
                    dec.ExpressionMappings.AddExpression(alias, expr);
                    dec.ExpressionMappings.MapExpression(alias, dstProp, createIfNotExists);

                    FdoDataPropertyConversionRule rule = task.GetDataConversionRule(alias);
                    PropertyConversionNodeDecorator cd = dec.ExpressionMappings.GetConversionRule(alias);

                    if (rule != null)
                    {
                        cd.NullOnFailedConversion = rule.NullOnFailure;
                        cd.Truncate = rule.Truncate;
                    }
                }
            }
        }
        public override int Execute()
        {
            CommandStatus retCode;

            FdoConnection srcConn = new FdoConnection(_srcProvider, _srcConnStr);
            FdoConnection destConn = null;
            //Directory given, assume SHP
            if (Directory.Exists(_destPath))
            {
                destConn = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + _destPath);
            }
            else
            {
                if (ExpressUtility.CreateFlatFileDataSource(_destPath))
                    destConn = ExpressUtility.CreateFlatFileConnection(_destPath);
                else
                    throw new FdoException("Could not create data source: " + _destPath);
            }

            try
            {
                srcConn.Open();
                destConn.Open();

                string srcName = "SOURCE";
                string dstName = "TARGET";

                FdoBulkCopyOptions options = new FdoBulkCopyOptions();
                options.RegisterConnection(srcName, srcConn);
                options.RegisterConnection(dstName, destConn);
                using (FdoFeatureService srcService = srcConn.CreateFeatureService())
                using (FdoFeatureService destService = destConn.CreateFeatureService())
                {
                    //See if spatial context needs to be copied to target
                    if (!string.IsNullOrEmpty(_srcSpatialContext))
                    {
                        SpatialContextInfo srcCtx = srcService.GetSpatialContext(_srcSpatialContext);
                        if (srcCtx != null)
                        {
                            Console.WriteLine("Copying spatial context: " + srcCtx.Name);
                            ExpressUtility.CopyAllSpatialContexts(new SpatialContextInfo[] { srcCtx }, destConn, true);
                        }
                    }
                    else
                    {
                        //Copy all
                        ExpressUtility.CopyAllSpatialContexts(srcConn, destConn, true);
                    }

                    FeatureSchema srcSchema = null;
                    //See if partial class list is needed
                    if (_srcClasses.Count > 0)
                    {
                        WriteLine("Checking if partial schema discovery is supported: " + srcService.SupportsPartialSchemaDiscovery());

                        srcSchema = srcService.PartialDescribeSchema(_srcSchema, _srcClasses);
                    }
                    else //Full copy
                    {
                        WriteLine("No classes specified, reading full source schema");
                        srcSchema = srcService.GetSchemaByName(_srcSchema);
                    }

                    if (srcSchema == null)
                    {
                        WriteError("Could not find source schema: " + _srcSchema);
                        retCode = CommandStatus.E_FAIL_SCHEMA_NOT_FOUND;
                    }
                    else
                    {
                        WriteLine("Checking source schema for incompatibilities");
                        FeatureSchema targetSchema = null;
                        IncompatibleSchema incSchema;
                        if (destService.CanApplySchema(srcSchema, out incSchema))
                        {
                            int clsCount = srcSchema.Classes.Count;
                            WriteLine("Applying source schema (containing " +  clsCount + " classes) to target");
                            destService.ApplySchema(srcSchema, null, true);
                            targetSchema = srcSchema;
                        }
                        else
                        {
                            WriteWarning("Incompatibilities were detected in source schema. Applying a modified version to target");
                            FeatureSchema fixedSchema = destService.AlterSchema(srcSchema, incSchema);
                            int clsCount = fixedSchema.Classes.Count;
                            WriteLine("Applying modified source schema (containing " + clsCount + " classes) to target");
                            destService.ApplySchema(fixedSchema, null, true);
                            targetSchema = fixedSchema;
                        }

                        //Now set class copy options
                        foreach (ClassDefinition cd in srcSchema.Classes)
                        {
                            FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, srcSchema.Name, cd.Name, targetSchema.Name, cd.Name);
                            copt.FlattenGeometries = _flatten;
                            options.AddClassCopyOption(copt);
                        }

                        if (_flatten)
                        {
                            WriteWarning("The switch -flatten has been defined. Geometries that are copied will have any Z or M coordinates removed");
                        }

                        FdoBulkCopy copy = new FdoBulkCopy(options);
                        copy.ProcessMessage += new MessageEventHandler(OnMessage);
                        copy.ProcessCompleted += new EventHandler(OnCompleted);
                        Console.WriteLine("Executing bulk copy");
                        copy.Execute();
                        List<Exception> errors = new List<Exception>(copy.GetAllErrors());
                        if (errors.Count > 0)
                        {
                            string file = GenerateLogFileName("bcp-error-");
                            LogErrors(errors, file);
                            base.WriteError("Errors were encountered during bulk copy.");
                            retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS;
                        }
                        else { retCode = CommandStatus.E_OK; }
                        retCode = CommandStatus.E_OK;
                    }
                }
            }
            catch (Exception ex)
            {
                WriteException(ex);
                retCode = CommandStatus.E_FAIL_UNKNOWN;
            }
            finally
            {
                srcConn.Dispose();
                destConn.Dispose();
            }
            return (int)retCode;
        }
Example #3
0
 /// <summary>
 /// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
 /// </summary>
 /// <param name="options">The options.</param>
 public FdoBulkCopy(FdoBulkCopyOptions options)
 {
     _options = options;
 }
Example #4
0
 /// <summary>
 /// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
 /// </summary>
 /// <param name="options">The options.</param>
 /// <param name="reportFrequency">The report frequency.</param>
 public FdoBulkCopy(FdoBulkCopyOptions options, int reportFrequency)
     : this(options)
 {
     _ReportFrequency = reportFrequency;
 }
Example #5
0
        /// <summary>
        /// Utility method to create a bulk copy operation from
        /// one class to another
        /// </summary>
        /// <param name="sourceConn"></param>
        /// <param name="targetConn"></param>
        /// <param name="srcSchemaName"></param>
        /// <param name="srcQuery"></param>
        /// <param name="targetSchemaName"></param>
        /// <param name="targetClassName"></param>
        /// <param name="propertyMapping"></param>
        /// <returns></returns>
        public static FdoBulkCopy CreateBulkCopy(
            FdoConnection sourceConn, 
            FdoConnection targetConn, 
            string srcSchemaName,
            FeatureQueryOptions srcQuery, 
            string targetSchemaName,
            string targetClassName, 
            NameValueCollection propertyMapping)
        {
            var dict = new Dictionary<string, FdoConnection>();
            dict["SOURCE"] = sourceConn;
            dict["TARGET"] = targetConn;

            var opts = new FdoBulkCopyOptions(dict, false);
            var copt = new FdoClassCopyOptions(
                "SOURCE",
                "TARGET",
                srcSchemaName,
                srcQuery.ClassName,
                targetSchemaName,
                targetClassName);

            if (!string.IsNullOrEmpty(srcQuery.Filter))
                copt.SourceFilter = srcQuery.Filter;

            foreach (string p in propertyMapping.Keys)
            {
                copt.AddPropertyMapping(p, propertyMapping[p]);
            }

            copt.FlattenGeometries = true;
            copt.ForceWkb = true;

            opts.AddClassCopyOption(copt);
            return new FdoBulkCopy(opts, 100);
        }
Example #6
0
        /// <summary>
        /// Creates a FDO bulk copy task. The target file will be created as part of 
        /// this method call. If the target path is a directory, it is assumed that
        /// SHP files are to be created and copied to.
        /// </summary>
        /// <param name="sourceFile">The path to the source file.</param>
        /// <param name="targetPath">
        /// The path to the target file/directory. If it is a directory, it is assumed
        /// that SHP files are to be created and copied to.
        /// </param>
        /// <param name="copySpatialContexts">If true, will also copy spatial contexts</param>
        /// <param name="fixIncompatibleSchema">If true, will try to fix the source schema to make it compatible with the target connection. If false, an exception will be thrown</param>
        /// <param name="flattenGeometries">If true, will strip all Z and M coordinates from all geometries that are copied</param>
        /// <returns></returns>
        public static FdoBulkCopy CreateBulkCopy(string sourceFile, string targetPath, bool copySpatialContexts, bool fixIncompatibleSchema, bool flattenGeometries)
        {
            FdoBulkCopyOptions options = null;
            FdoConnection source = null;
            FdoConnection target = null;

            try
            {
                //Is a directory. Implies a SHP connection
                if (IsShp(targetPath))
                {
                    //SHP doesn't actually support CreateDataStore. We use the following technique:
                    // - Connect to base directory
                    // - Clone source schema and apply to SHP connection.
                    // - A SHP file and related files are created for each feature class.
                    string shpdir = Directory.Exists(targetPath) ? targetPath : Path.GetDirectoryName(targetPath);
                    source = CreateFlatFileConnection(sourceFile);
                    target = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + shpdir);

                    source.Open();

                    //Verify source has only classes with single geometry storage and only one geometry
                    using (FdoFeatureService svc = source.CreateFeatureService())
                    {
                        using (FeatureSchemaCollection schemas = svc.DescribeSchema())
                        {
                            foreach (FeatureSchema sch in schemas)
                            {
                                foreach (ClassDefinition cd in sch.Classes)
                                {
                                    int geomProps = 0;
                                    foreach (PropertyDefinition pd in cd.Properties)
                                    {
                                        if (pd.PropertyType == PropertyType.PropertyType_GeometricProperty)
                                        {
                                            GeometricPropertyDefinition gp = pd as GeometricPropertyDefinition;
                                            GeometricType[] types = FdoGeometryUtil.GetGeometricTypes(gp.GeometryTypes);
                                            if (types.Length != 1 || (types.Length == 1 && types[0] == GeometricType.GeometricType_All))
                                                throw new FdoETLException(string.Format("Source file cannot be copied to a SHP file. {0}:{1}.{2} has more than one geometry storage type", sch.Name, cd.Name, pd.Name));
                                            geomProps++;
                                        }
                                    }
                                    if (geomProps > 1)
                                        throw new FdoETLException("Source file cannot be copied to a SHP file. One or more feature classes have more than one geometry property");
                                }
                            }
                        }
                    }
                }
                else
                {
                    if (!CreateFlatFileDataSource(targetPath))
                        throw new FdoException("Unable to create data source on: " + targetPath);
                    source = CreateFlatFileConnection(sourceFile);
                    target = CreateFlatFileConnection(targetPath);
                }

                //Source and target connections may have been opened before this point
                if (source.State == FdoConnectionState.Closed)
                    source.Open();

                if (target.State == FdoConnectionState.Closed)
                    target.Open();

                string srcName = "SOURCE";
                string dstName = "TARGET";

                Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>();
                connections.Add(srcName, source);
                connections.Add(dstName, target);

                options = new FdoBulkCopyOptions(connections, true);

                if (copySpatialContexts)
                {
                    CopyAllSpatialContexts(source, target, true);
                }

                using (FdoFeatureService srcService = source.CreateFeatureService())
                using (FdoFeatureService destService = target.CreateFeatureService())
                {
                    FeatureSchemaCollection schemas = srcService.DescribeSchema();
                    //Assume single-schema
                    FeatureSchema fs = schemas[0];
                    //Clone and apply to target
                    FeatureSchema targetSchema = FdoSchemaUtil.CloneSchema(fs);
                    IncompatibleSchema incSchema;
                    string sourceSchemaName = fs.Name;
                    string targetSchemaName = string.Empty;

                    bool canApply = destService.CanApplySchema(targetSchema, out incSchema);
                    if (canApply)
                    {
                        destService.ApplySchema(targetSchema);
                        targetSchemaName = targetSchema.Name;
                    }
                    else
                    {
                        if (fixIncompatibleSchema)
                        {
                            FeatureSchema fixedSchema = destService.AlterSchema(targetSchema, incSchema);
                            destService.ApplySchema(fixedSchema);
                            targetSchemaName = fixedSchema.Name;
                        }
                        else
                        {
                            throw new Exception(incSchema.ToString());
                        }
                    }

                    //Copy all classes
                    foreach (ClassDefinition cd in fs.Classes)
                    {
                        FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, sourceSchemaName, cd.Name, targetSchemaName, cd.Name);
                        copt.Name = "Copy source to target [" + cd.Name + "]";
                        copt.FlattenGeometries = flattenGeometries;
                        options.AddClassCopyOption(copt);

                        //Flick on batch support if we can
                        if (destService.SupportsBatchInsertion())
                            copt.BatchSize = 300; //Madness? THIS IS SPARTA!
                    }
                }
            }
            catch (Exception)
            {
                if (source != null)
                    source.Dispose();
                if (target != null)
                    target.Dispose();

                throw;
            }
            return new FdoBulkCopy(options);
        }
Example #7
0
        /// <summary>
        /// Loads bulk copy options from deserialized xml
        /// </summary>
        /// <param name="def">The deserialized definition.</param>
        /// <param name="name">The name.</param>
        /// <param name="owner">if set to <c>true</c> [owner].</param>
        /// <returns></returns>
        public FdoBulkCopyOptions BulkCopyFromXml(FdoBulkCopyTaskDefinition def, ref string name, bool owner)
        {
            var nameMap = Prepare(def);

            // TODO/FIXME/HACK:
            //
            // The introduction of on-the-fly schema modifications before copying has introduced a
            // potential problem which will only occur when multiple copy tasks copy to the same target
            // feature class. 
            //
            // What happens is because a target will be multiple sources copying to it, if we need
            // to create/update the target class definition, the current infrastructure will be confused
            // as to which source feature class to clone from.
            //
            // A possible solution is to return the first matching name. This will cause a "create" operation
            // to be queued. We then alter the pre-modification process so that subsequent "create" operations
            // becomes "update" operations instead.
            //
            // On second thought, the current infrastructure should be smart enought to prevent this, as
            // the UI only allows:
            //
            // 1. Mapping to an existing class
            // 2. Mapping to a class of the same name (create if necessary)
            //
            // It won't be possible to map to a class that doesn't exist, as the requirement for auto-create/update
            // is that the class name will be the same.
            //
            // ie. It won't be possible to create this mapping
            //
            // a. Foo -> Foo (create if necessary)
            // b. Bar -> Foo
            // 
            // Rule 1 prevents mapping b) from happening
            // Rule 2 ensures that Foo will only ever be created once
            //
            // If multiple sources are mapped to the same target, I believe the bcp infrastructure should ensure
            // that target must already exist first.


            name = def.name;
            Dictionary<string, FdoConnection> connections = new Dictionary<string, FdoConnection>();
            Dictionary<string, string> changeConnNames = new Dictionary<string, string>();

            //TODO: Prepare() ensured that all connections have unique names that don't already exist
            //now in the effort to re-use existing connections, see which connection entries
            //already exist and can be renamed back to the old name

            foreach (FdoConnectionEntryElement entry in def.Connections)
            {
                string connName = entry.name;
                FdoConnection conn = CreateConnection(entry.provider, entry.ConnectionString, entry.configPath, ref connName);

                connections[connName] = conn;

                if (connName != entry.name)
                {
                    changeConnNames[entry.name] = connName;
                }
            }
            
            foreach (string oldName in changeConnNames.Keys)
            {
                def.UpdateConnectionReferences(oldName, changeConnNames[oldName]);
            }

            //Compile the list of classes to be queried
            Dictionary<string, MultiSchemaQuery> queries = new Dictionary<string, MultiSchemaQuery>();
            foreach (FdoCopyTaskElement task in def.CopyTasks)
            {
                MultiSchemaQuery src = null;
                MultiSchemaQuery dst = null;

                //Process source
                if (!queries.ContainsKey(task.Source.connection))
                    src = queries[task.Source.connection] = new MultiSchemaQuery(task.Source.connection, SchemaOrigin.Source);
                else
                    src = queries[task.Source.connection];

                var sq = src.TryGet(task.Source.schema);
                if (sq != null)
                {
                    sq.AddClass(task.Source.@class);
                }
                else
                {
                    sq = new SchemaQuery(task.Source.schema);
                    sq.AddClass(task.Source.@class);
                    src.Add(sq);
                }

                //Process target
                if (!queries.ContainsKey(task.Target.connection))
                    dst = queries[task.Target.connection] = new MultiSchemaQuery(task.Target.connection, SchemaOrigin.Target);
                else
                    dst = queries[task.Target.connection];

                var tq = dst.TryGet(task.Target.schema);
                if (tq != null)
                {
                    tq.AddClass(task.Target.@class);
                }
                else
                {
                    tq = new SchemaQuery(task.Target.schema);
                    tq.AddClass(task.Target.@class);
                    dst.Add(tq);
                }
            }

            List<TargetClassModificationItem> modifiers = new List<TargetClassModificationItem>();
            using (var schemaCache = new FeatureSchemaCache())
            {
                //Now populate the schema cache with source schemas
                foreach (string connName in queries.Keys)
                {
                    if (connections.ContainsKey(connName))
                    {
                        var mqry = queries[connName];
                        var conn = connections[connName];

                        FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);

                        using (var svc = conn.CreateFeatureService())
                        {
                            foreach (var sq in mqry.SchemaQueries)
                            {
                                //Source schema queries are expected to fully check out so use original method
                                if (mqry.Origin == SchemaOrigin.Source)
                                {
                                    schemas.Add(svc.PartialDescribeSchema(sq.SchemaName, new List<string>(sq.ClassNames)));
                                }
                            }
                        }

                        if (schemas.Count > 0)
                            schemaCache.Add(connName, schemas);
                    }
                }

                //Now populate the schema cache with target schemas, taking note of any
                //classes that need to be created or updated.
                foreach (string connName in queries.Keys)
                {
                    if (connections.ContainsKey(connName))
                    {
                        var mqry = queries[connName];
                        var conn = connections[connName];

                        FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);

                        using (var svc = conn.CreateFeatureService())
                        {
                            foreach (var sq in mqry.SchemaQueries)
                            {
                                //Source schema queries are expected to fully check out so use original method
                                if (mqry.Origin == SchemaOrigin.Target)
                                {
                                    //Because we may need to create new classes, the old method will break down on non-existent class names
                                    //So use the new method
                                    string[] notFound;
                                    var schema = svc.PartialDescribeSchema(sq.SchemaName, new List<string>(sq.ClassNames), out notFound);
                                    //if (notFound.Length > 0)
                                    //{
                                    //    //If we can't modify schemas we'll stop right here. This is caused by elements containing createIfNotExists = true
                                    //    if (!conn.Capability.GetBooleanCapability(CapabilityType.FdoCapabilityType_SupportsSchemaModification))
                                    //        throw new NotSupportedException("The connection named " + connName + " does not support schema modification. Therefore, copy tasks and property/expression mappings cannot have createIfNotExists = true");

                                    //    //This cumbersome, but we need the parent schema name of the class that we will need to copy
                                    //    string srcSchema = GetSourceSchemaForMapping(def, sq.SchemaName, notFound);

                                    //    foreach (string className in notFound)
                                    //    {
                                    //        modifiers.Add(new CreateTargetClassFromSource(srcSchema, className));
                                    //    }
                                    //}
                                    schemas.Add(schema);
                                }
                            }
                        }

                        if (schemas.Count > 0)
                            schemaCache.Add(connName, schemas);
                    }
                }

                FdoBulkCopyOptions opts = new FdoBulkCopyOptions(connections, owner);
                
                foreach (FdoCopyTaskElement task in def.CopyTasks)
                {
                    TargetClassModificationItem mod;
                    FdoClassCopyOptions copt = FdoClassCopyOptions.FromElement(task, schemaCache, connections[task.Source.connection], connections[task.Target.connection], out mod);
                    opts.AddClassCopyOption(copt);

                    if (mod != null)
                        copt.PreCopyTargetModifier = mod;
                        //opts.AddClassModifier(mod);
                }

                return opts;
            }
        }
Example #8
0
 /// <summary>
 /// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
 /// </summary>
 /// <param name="options">The options.</param>
 /// <param name="reportFrequency">The report frequency.</param>
 public FdoBulkCopy(FdoBulkCopyOptions options, int reportFrequency)
     : this(options)
 {
     _ReportFrequency = reportFrequency;
 }
Example #9
0
 /// <summary>
 /// Initializes a new instance of the <see cref="FdoBulkCopy"/> class.
 /// </summary>
 /// <param name="options">The options.</param>
 public FdoBulkCopy(FdoBulkCopyOptions options)
 {
     _options = options;
 }