A specialized ETL process that consists of one or more FdoClassToClassCopyProcess instances
Inheritance: FdoSpecializedEtlProcess
Exemplo n.º 1
0
 public FdoBulkCopyCtl(string name, FdoBulkCopy task)
     : this()
 {
     Load(task.Options, name);
     txtName.ReadOnly = true;
     btnSave.Enabled = true;
 }
Exemplo n.º 2
0
        public void Load()
        {
            TaskLoader ldr = new TaskLoader();
            string path = Preferences.SessionDirectory;
            if (System.IO.Directory.Exists(path))
            {
                string[] files = System.IO.Directory.GetFiles(path, "*" + TaskDefinitionHelper.BULKCOPYDEFINITION);
                foreach (string f in files)
                {
                    try
                    {
                        string name = string.Empty;
                        FdoBulkCopyOptions opt = ldr.BulkCopyFromXml(f, ref name, false);
                        FdoBulkCopy cpy = new FdoBulkCopy(opt);
                        AddTask(name, cpy);
                    }
                    catch { }
                }
                files = System.IO.Directory.GetFiles(path, "*" + TaskDefinitionHelper.JOINDEFINITION);
                foreach (string f in files)
                {
                    try
                    {
                        string name = string.Empty;
                        FdoJoinOptions opt = ldr.JoinFromXml(f, ref name, false);
                        FdoJoin join = new FdoJoin(opt);
                        AddTask(name, join);
                    }
                    catch { }
                }

                files = System.IO.Directory.GetFiles(path, "*" + TaskDefinitionHelper.SEQUENTIALPROCESS);
                foreach (string f in files)
                {
                    try
                    {
                        string prefix = Path.GetFileNameWithoutExtension(f);
                        string name = prefix;
                        int counter = 0;
                        while (this.NameExists(name))
                        {
                            counter++;
                            name = prefix + counter;
                        }
                        SequentialProcessDefinition spd = (SequentialProcessDefinition)SequentialProcessDefinition.Serializer.Deserialize(File.OpenRead(f));
                        FdoSequentialProcess proc = new FdoSequentialProcess(spd);
                        AddTask(name, proc);
                    }
                    catch { }
                }
            }
        }
Exemplo n.º 3
0
 /// <summary>
 /// Handles the file drop
 /// </summary>
 /// <param name="file">The file being dropped</param>
 public void HandleDrop(string file)
 {
     TaskManager mgr = ServiceManager.Instance.GetService<TaskManager>();
     TaskLoader ldr = new TaskLoader();
     string prefix = string.Empty;
     FdoBulkCopyOptions opt = ldr.BulkCopyFromXml(file, ref prefix, false);
     FdoBulkCopy cpy = new FdoBulkCopy(opt);
     
     string name = prefix;
     int counter = 0;
     while (mgr.NameExists(name))
     {
         counter++;
         name = prefix + counter;
     }
     mgr.AddTask(name, cpy);
 }
Exemplo n.º 4
0
        public override int Execute()
        {
            CommandStatus retCode;

            FdoConnection srcConn = new FdoConnection(_srcProvider, _srcConnStr);
            FdoConnection destConn = null;
            //Directory given, assume SHP
            if (Directory.Exists(_destPath))
            {
                destConn = new FdoConnection("OSGeo.SHP", "DefaultFileLocation=" + _destPath);
            }
            else
            {
                if (ExpressUtility.CreateFlatFileDataSource(_destPath))
                    destConn = ExpressUtility.CreateFlatFileConnection(_destPath);
                else
                    throw new FdoException("Could not create data source: " + _destPath);
            }

            try
            {
                srcConn.Open();
                destConn.Open();

                string srcName = "SOURCE";
                string dstName = "TARGET";

                FdoBulkCopyOptions options = new FdoBulkCopyOptions();
                options.RegisterConnection(srcName, srcConn);
                options.RegisterConnection(dstName, destConn);
                using (FdoFeatureService srcService = srcConn.CreateFeatureService())
                using (FdoFeatureService destService = destConn.CreateFeatureService())
                {
                    //See if spatial context needs to be copied to target
                    if (!string.IsNullOrEmpty(_srcSpatialContext))
                    {
                        SpatialContextInfo srcCtx = srcService.GetSpatialContext(_srcSpatialContext);
                        if (srcCtx != null)
                        {
                            Console.WriteLine("Copying spatial context: " + srcCtx.Name);
                            ExpressUtility.CopyAllSpatialContexts(new SpatialContextInfo[] { srcCtx }, destConn, true);
                        }
                    }
                    else
                    {
                        //Copy all
                        ExpressUtility.CopyAllSpatialContexts(srcConn, destConn, true);
                    }

                    FeatureSchema srcSchema = null;
                    //See if partial class list is needed
                    if (_srcClasses.Count > 0)
                    {
                        WriteLine("Checking if partial schema discovery is supported: " + srcService.SupportsPartialSchemaDiscovery());

                        srcSchema = srcService.PartialDescribeSchema(_srcSchema, _srcClasses);
                    }
                    else //Full copy
                    {
                        WriteLine("No classes specified, reading full source schema");
                        srcSchema = srcService.GetSchemaByName(_srcSchema);
                    }

                    if (srcSchema == null)
                    {
                        WriteError("Could not find source schema: " + _srcSchema);
                        retCode = CommandStatus.E_FAIL_SCHEMA_NOT_FOUND;
                    }
                    else
                    {
                        WriteLine("Checking source schema for incompatibilities");
                        FeatureSchema targetSchema = null;
                        IncompatibleSchema incSchema;
                        if (destService.CanApplySchema(srcSchema, out incSchema))
                        {
                            int clsCount = srcSchema.Classes.Count;
                            WriteLine("Applying source schema (containing " +  clsCount + " classes) to target");
                            destService.ApplySchema(srcSchema, null, true);
                            targetSchema = srcSchema;
                        }
                        else
                        {
                            WriteWarning("Incompatibilities were detected in source schema. Applying a modified version to target");
                            FeatureSchema fixedSchema = destService.AlterSchema(srcSchema, incSchema);
                            int clsCount = fixedSchema.Classes.Count;
                            WriteLine("Applying modified source schema (containing " + clsCount + " classes) to target");
                            destService.ApplySchema(fixedSchema, null, true);
                            targetSchema = fixedSchema;
                        }

                        //Now set class copy options
                        foreach (ClassDefinition cd in srcSchema.Classes)
                        {
                            FdoClassCopyOptions copt = new FdoClassCopyOptions(srcName, dstName, srcSchema.Name, cd.Name, targetSchema.Name, cd.Name);
                            copt.FlattenGeometries = _flatten;
                            options.AddClassCopyOption(copt);
                        }

                        if (_flatten)
                        {
                            WriteWarning("The switch -flatten has been defined. Geometries that are copied will have any Z or M coordinates removed");
                        }

                        FdoBulkCopy copy = new FdoBulkCopy(options);
                        copy.ProcessMessage += new MessageEventHandler(OnMessage);
                        copy.ProcessCompleted += new EventHandler(OnCompleted);
                        Console.WriteLine("Executing bulk copy");
                        copy.Execute();
                        List<Exception> errors = new List<Exception>(copy.GetAllErrors());
                        if (errors.Count > 0)
                        {
                            string file = GenerateLogFileName("bcp-error-");
                            LogErrors(errors, file);
                            base.WriteError("Errors were encountered during bulk copy.");
                            retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS;
                        }
                        else { retCode = CommandStatus.E_OK; }
                        retCode = CommandStatus.E_OK;
                    }
                }
            }
            catch (Exception ex)
            {
                WriteException(ex);
                retCode = CommandStatus.E_FAIL_UNKNOWN;
            }
            finally
            {
                srcConn.Dispose();
                destConn.Dispose();
            }
            return (int)retCode;
        }
Exemplo n.º 5
0
        private void btnSave_Click(object sender, EventArgs e)
        {
            if (mTreeView.Nodes[0].Nodes.Count == 0)
            {
                MessageService.ShowMessage("Please define at least one copy task");
                return;
            }

            if (string.IsNullOrEmpty(txtName.Text))
            {
                MessageService.ShowMessage("Please specify a name for this task");
                return;
            }

            TaskManager tmgr = ServiceManager.Instance.GetService<TaskManager>();

            if (IsNew && tmgr.GetTask(txtName.Text) != null)
            {
                MessageService.ShowMessage("A task named " + txtName.Text + " already exists. Please specify a name for this task");
                return;
            }

            //This could take a while...
            using (new TempCursor(Cursors.WaitCursor))
            {
                LoggingService.Info("Updating loaded task. Please wait.");
                TaskLoader loader = new TaskLoader();
                if (IsNew)
                {
                    string name = string.Empty;
                    FdoBulkCopyOptions opts = loader.BulkCopyFromXml(Save(), ref name, false);
                    FdoBulkCopy bcp = new FdoBulkCopy(opts);
                    tmgr.AddTask(name, bcp);
                    this.Close();
                }
                else
                {
                    FdoBulkCopy bcp = tmgr.GetTask(txtName.Text) as FdoBulkCopy;
                    if (bcp == null)
                    {
                        MessageService.ShowMessage("This named task is not a bulk copy task or could not find the named task to update");
                        return;
                    }
                    string name = string.Empty;
                    FdoBulkCopyOptions opts = loader.BulkCopyFromXml(Save(), ref name, false);
                    Debug.Assert(name == txtName.Text); //unchanged

                    //Update options
                    bcp.Options = opts;
                    MessageService.ShowMessage("Task updated. To save to disk, right-click the task object and choose: " + ResourceService.GetString("CMD_SaveTask"));
                    this.Close();
                }
            }
        }
Exemplo n.º 6
0
        public override void Run()
        {
            TaskManager mgr = ServiceManager.Instance.GetService<TaskManager>();
            TaskLoader ldr = new TaskLoader();
            string file = FileService.OpenFile(ResourceService.GetString("TITLE_LOAD_TASK"), ResourceService.GetString("FILTER_TASK_DEFINITION"));
            if (FileService.FileExists(file))
            {
                using (new TempCursor(Cursors.WaitCursor))
                {
                    LoggingService.Info(ResourceService.GetString("LOADING_TASK_DEFINITION_WAIT"));
                    if (TaskDefinitionHelper.IsBulkCopy(file))
                    {
                        string name = string.Empty;
                        FdoBulkCopyOptions opt = ldr.BulkCopyFromXml(file, ref name, false);
                        FdoBulkCopy cpy = new FdoBulkCopy(opt);
                        mgr.AddTask(name, cpy);
                    }
                    else if (TaskDefinitionHelper.IsJoin(file))
                    {
                        string name = string.Empty;
                        FdoJoinOptions opt = ldr.JoinFromXml(file, ref name, false);
                        FdoJoin join = new FdoJoin(opt);
                        mgr.AddTask(name, join);
                    }
                    else if (TaskDefinitionHelper.IsSequentialProcess(file))
                    {
                        using (var fs = File.OpenRead(file))
                        {
                            int counter = 0;
                            var prefix = Path.GetFileNameWithoutExtension(file);
                            var name = prefix;
                            while (mgr.NameExists(name))
                            {
                                counter++;
                                name = prefix + counter;
                            }

                            var def = (SequentialProcessDefinition)SequentialProcessDefinition.Serializer.Deserialize(fs);
                            var proc = new FdoSequentialProcess(def);
                            mgr.AddTask(name, proc);
                        }
                    }
                }
            }
        }
Exemplo n.º 7
0
        public override int Execute()
        {
            CommandStatus retCode;
            DefinitionLoader loader = new DefinitionLoader();
            string name = null;
            if (TaskDefinitionHelper.IsBulkCopy(_file))
            {
                FdoBulkCopyTaskDefinition def = null;
                using (var sr = new StreamReader(_file))
                {
                    XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition));
                    def = (FdoBulkCopyTaskDefinition)ser.Deserialize(sr);
                }

                if (def == null)
                {
                    return (int)CommandStatus.E_FAIL_TASK_VALIDATION;
                }

                //If more than one task specified, load the default task and weed
                //out unneeded elements.
                if (_taskNames.Length > 0)
                {
                    base.WriteLine("Certain tasks have been specified, only part of the bulk copy definition will execute");
                    var keepConnections = new Dictionary<string, FdoConnectionEntryElement>();
                    var keepTasks = new List<FdoCopyTaskElement>();

                    //Store needed tasks
                    foreach (var task in def.CopyTasks)
                    {
                        if (Array.IndexOf(_taskNames, task.name) >= 0)
                        {
                            keepTasks.Add(task);
                        }
                    }

                    //Store needed connections
                    foreach (var task in keepTasks)
                    {
                        foreach (var conn in def.Connections)
                        {
                            //Is referenced as source/target connection?
                            if (task.Source.connection == conn.name || task.Target.connection == conn.name)
                            {
                                if (!keepConnections.ContainsKey(conn.name))
                                    keepConnections.Add(conn.name, conn);
                            }
                        }
                    }

                    if (keepTasks.Count != _taskNames.Length)
                    {
                        List<string> names = new List<string>();
                        foreach (var n in _taskNames)
                        {
                            bool found = false;
                            foreach (var task in keepTasks)
                            {
                                if (task.name == n)
                                {
                                    found = true;
                                    break;
                                }
                            }

                            if (!found)
                                names.Add(n);
                        }

                        base.WriteError("Could not find specified tasks in bulk copy definition: " + string.Join(",", names.ToArray()));

                        return (int)CommandStatus.E_FAIL_MISSING_BULK_COPY_TASKS;
                    }

                    //Now replace
                    def.Connections = new List<FdoConnectionEntryElement>(keepConnections.Values).ToArray();
                    def.CopyTasks = keepTasks.ToArray();
                }

                using (FdoBulkCopyOptions opts = loader.BulkCopyFromXml(def, ref name, true))
                {
                    FdoBulkCopy copy = new FdoBulkCopy(opts);
                    copy.ProcessMessage += delegate(object sender, MessageEventArgs e)
                    {
                        base.WriteLine(e.Message);
                    };
                    copy.ProcessAborted += delegate(object sender, EventArgs e)
                    {
                        base.WriteLine("Bulk Copy Aborted");
                    };
                    copy.ProcessCompleted += delegate(object sender, EventArgs e)
                    {
                        base.WriteLine("Bulk Copy Completed");
                    };
                    copy.Execute();
                    List<Exception> errors = new List<Exception>(copy.GetAllErrors());
                    if (errors.Count > 0)
                    {
                        string file = GenerateLogFileName("bcp-error-");
                        LogErrors(errors, file);
                        base.WriteError("Errors were encountered during bulk copy.");
                        retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS;
                    }
                    else { retCode = CommandStatus.E_OK; }
                }
            }
            else if (TaskDefinitionHelper.IsJoin(_file))
            {
                if (_taskNames.Length > 0)
                {
                    base.WriteError("Parameter -bcptask is not applicable for join tasks");
                    return (int)CommandStatus.E_FAIL_INVALID_ARGUMENTS;
                }

                using (FdoJoinOptions opts = loader.JoinFromXml(_file, ref name, true))
                {
                    opts.Left.Connection.Open();
                    opts.Right.Connection.Open();
                    opts.Target.Connection.Open();
                    FdoJoin join = new FdoJoin(opts);
                    join.ProcessMessage += delegate(object sender, MessageEventArgs e)
                    {
                        base.WriteLine(e.Message);
                    };
                    join.Execute();
                    List<Exception> errors = new List<Exception>(join.GetAllErrors());
                    if (errors.Count > 0)
                    {
                        string file = GenerateLogFileName("join-error-");
                        LogErrors(errors, file);
                        base.WriteError("Errors were encountered during join operation");
                        retCode = CommandStatus.E_FAIL_JOIN_WITH_ERRORS;
                    }
                    else { retCode = CommandStatus.E_OK; }
                }
            }
            else if (TaskDefinitionHelper.IsSequentialProcess(_file))
            {
                var def = (SequentialProcessDefinition)SequentialProcessDefinition.Serializer.Deserialize(File.OpenRead(_file));
                var proc = new FdoSequentialProcess(def);
                proc.ProcessMessage += delegate(object sender, MessageEventArgs e)
                {
                    base.WriteLine(e.Message);
                };
                proc.Execute();
                List<Exception> errors = new List<Exception>(proc.GetAllErrors());
                if (errors.Count > 0)
                {
                    string file = GenerateLogFileName("seq-process-");
                    LogErrors(errors, file);
                    base.WriteError("Errors were encountered during sequential process");
                }
                //Why E_OK? the user should check the log for the underlying return codes
                //of individual FdoUtil.exe invocations!
                retCode = CommandStatus.E_OK;
            }
            else
            {
                retCode = CommandStatus.E_FAIL_UNRECOGNISED_TASK_FORMAT;
            }

            return (int)retCode;
        }