/// <summary> /// Loads bulk copy options from xml /// </summary> /// <param name="file">The file.</param> /// <param name="name">The name.</param> /// <param name="owner">if set to <c>true</c> [owner].</param> /// <returns></returns> public FdoBulkCopyOptions BulkCopyFromXml(string file, ref string name, bool owner) { FdoBulkCopyTaskDefinition def = null; XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition)); def = (FdoBulkCopyTaskDefinition)ser.Deserialize(new StreamReader(file)); return(BulkCopyFromXml(def, ref name, owner)); }
/// <summary> /// Saves this process to a file /// </summary> /// <param name="file">The file to save this process to</param> /// <param name="name">The name of the process</param> public override void Save(string file, string name) { FdoBulkCopyTaskDefinition def = new FdoBulkCopyTaskDefinition(); def.name = name; List <FdoConnectionEntryElement> connList = new List <FdoConnectionEntryElement>(); List <FdoCopyTaskElement> copyTasks = new List <FdoCopyTaskElement>(); foreach (string connName in _options.ConnectionNames) { FdoConnection conn = _options.GetConnection(connName); FdoConnectionEntryElement entry = new FdoConnectionEntryElement(); entry.name = connName; entry.provider = conn.Provider; entry.ConnectionString = conn.ConnectionString; if (conn.HasConfiguration) { string path = Path.GetDirectoryName(file); path = Path.Combine(path, entry.name + "_configuration.xml"); conn.SaveConfiguration(path); entry.configPath = path; } connList.Add(entry); } foreach (FdoClassCopyOptions copt in _options.ClassCopyOptions) { copyTasks.Add(copt.ToElement()); } def.Connections = connList.ToArray(); def.CopyTasks = copyTasks.ToArray(); using (StreamWriter writer = new StreamWriter(file, false)) { XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition)); ser.Serialize(writer, def); } }
private static string GetSourceSchemaForMapping(FdoBulkCopyTaskDefinition def, string targetSchema, string[] classNames) { ISet <string> matches = new HashSet <string>(); List <string> classes = new List <string>(classNames); foreach (var task in def.CopyTasks) { if (classes.Contains(task.Target.@class) && task.Target.schema.Equals(targetSchema)) { matches.Add(task.Source.schema); } } if (matches.Count > 1) { throw new TaskValidationException("The specified class names have various parent schema names"); } if (matches.Count == 0) { throw new TaskValidationException("Could not determine parent schema name for the given class names"); } return(new List <string>(matches)[0]); //The price to pay for targeting linq-less fx 2.0 }
private FdoBulkCopyTaskDefinition Save() { FdoBulkCopyTaskDefinition def = new FdoBulkCopyTaskDefinition(); def.name = txtName.Text; List <FdoConnectionEntryElement> conns = new List <FdoConnectionEntryElement>(); foreach (DataGridViewRow row in grdConnections.Rows) { FdoConnectionEntryElement entry = new FdoConnectionEntryElement(); entry.name = row.Cells[0].Value.ToString(); entry.provider = row.Cells[1].Value.ToString(); entry.ConnectionString = row.Cells[3].Value.ToString(); conns.Add(entry); } List <FdoCopyTaskElement> tasks = new List <FdoCopyTaskElement>(); foreach (CopyTaskNodeDecorator dec in _tasks.Values) { FdoCopyTaskElement task = new FdoCopyTaskElement(); task.name = dec.Name; task.createIfNotExists = dec.CreateIfNotExists; task.Source = new FdoCopySourceElement(); task.Target = new FdoCopyTargetElement(); task.Options = new FdoCopyOptionsElement(); List <FdoPropertyMappingElement> pmaps = new List <FdoPropertyMappingElement>(); List <FdoExpressionMappingElement> emaps = new List <FdoExpressionMappingElement>(); //Source task.Source.@class = dec.SourceClassName; task.Source.connection = dec.SourceConnectionName; task.Source.schema = dec.SourceSchemaName; //Target task.Target.@class = dec.TargetClassName; task.Target.connection = dec.TargetConnectionName; task.Target.schema = dec.TargetSchemaName; //Options task.Options.BatchSize = dec.Options.BatchSize.ToString(); task.Options.FlattenGeometries = dec.Options.Flatten; task.Options.FlattenGeometriesSpecified = true; task.Options.DeleteTarget = dec.Options.Delete; task.Options.Filter = dec.Options.SourceFilter; task.Options.ForceWKB = dec.Options.ForceWkb; task.Options.ForceWKBSpecified = true; //Property Mappings NameValueCollection mappings = dec.PropertyMappings.GetPropertyMappings(); foreach (string srcProp in mappings.Keys) { string dstProp = mappings[srcProp]; FdoPropertyMappingElement p = new FdoPropertyMappingElement(); p.source = srcProp; p.target = dstProp; PropertyConversionNodeDecorator conv = dec.PropertyMappings.GetConversionRule(p.source); p.nullOnFailedConversion = conv.NullOnFailedConversion; p.truncate = conv.Truncate; p.createIfNotExists = conv.CreateIfNotExists; pmaps.Add(p); } foreach (string alias in dec.ExpressionMappings.GetAliases()) { FdoExpressionMappingElement e = new FdoExpressionMappingElement(); e.alias = alias; ExpressionMappingInfo exMap = dec.ExpressionMappings.GetMapping(alias); e.Expression = exMap.Expression; e.target = exMap.TargetProperty; PropertyConversionNodeDecorator conv = dec.ExpressionMappings.GetConversionRule(e.alias); e.nullOnFailedConversion = conv.NullOnFailedConversion; e.truncate = conv.Truncate; e.createIfNotExists = conv.CreateIfNotExists; emaps.Add(e); } task.PropertyMappings = pmaps.ToArray(); task.ExpressionMappings = emaps.ToArray(); tasks.Add(task); } def.Connections = conns.ToArray(); def.CopyTasks = tasks.ToArray(); return(def); }
public override int Execute() { CommandStatus retCode; DefinitionLoader loader = new DefinitionLoader(); string name = null; if (TaskDefinitionHelper.IsBulkCopy(_file)) { FdoBulkCopyTaskDefinition def = null; using (var sr = new StreamReader(_file)) { XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition)); def = (FdoBulkCopyTaskDefinition)ser.Deserialize(sr); } if (def == null) { return((int)CommandStatus.E_FAIL_TASK_VALIDATION); } //If more than one task specified, load the default task and weed //out unneeded elements. if (_taskNames.Length > 0) { base.WriteLine("Certain tasks have been specified, only part of the bulk copy definition will execute"); var keepConnections = new Dictionary <string, FdoConnectionEntryElement>(); var keepTasks = new List <FdoCopyTaskElement>(); //Store needed tasks foreach (var task in def.CopyTasks) { if (Array.IndexOf(_taskNames, task.name) >= 0) { keepTasks.Add(task); } } //Store needed connections foreach (var task in keepTasks) { foreach (var conn in def.Connections) { //Is referenced as source/target connection? if (task.Source.connection == conn.name || task.Target.connection == conn.name) { if (!keepConnections.ContainsKey(conn.name)) { keepConnections.Add(conn.name, conn); } } } } if (keepTasks.Count != _taskNames.Length) { List <string> names = new List <string>(); foreach (var n in _taskNames) { bool found = false; foreach (var task in keepTasks) { if (task.name == n) { found = true; break; } } if (!found) { names.Add(n); } } base.WriteError("Could not find specified tasks in bulk copy definition: " + string.Join(",", names.ToArray())); return((int)CommandStatus.E_FAIL_MISSING_BULK_COPY_TASKS); } //Now replace def.Connections = new List <FdoConnectionEntryElement>(keepConnections.Values).ToArray(); def.CopyTasks = keepTasks.ToArray(); } using (FdoBulkCopyOptions opts = loader.BulkCopyFromXml(def, ref name, true)) { FdoBulkCopy copy = new FdoBulkCopy(opts); copy.ProcessMessage += delegate(object sender, MessageEventArgs e) { base.WriteLine(e.Message); }; copy.ProcessAborted += delegate(object sender, EventArgs e) { base.WriteLine("Bulk Copy Aborted"); }; copy.ProcessCompleted += delegate(object sender, EventArgs e) { base.WriteLine("Bulk Copy Completed"); }; copy.Execute(); List <Exception> errors = new List <Exception>(copy.GetAllErrors()); if (errors.Count > 0) { string file = GenerateLogFileName("bcp-error-"); LogErrors(errors, file); base.WriteError("Errors were encountered during bulk copy."); retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS; } else { retCode = CommandStatus.E_OK; } } } else if (TaskDefinitionHelper.IsJoin(_file)) { if (_taskNames.Length > 0) { base.WriteError("Parameter -bcptask is not applicable for join tasks"); return((int)CommandStatus.E_FAIL_INVALID_ARGUMENTS); } using (FdoJoinOptions opts = loader.JoinFromXml(_file, ref name, true)) { opts.Left.Connection.Open(); opts.Right.Connection.Open(); opts.Target.Connection.Open(); FdoJoin join = new FdoJoin(opts); join.ProcessMessage += delegate(object sender, MessageEventArgs e) { base.WriteLine(e.Message); }; join.Execute(); List <Exception> errors = new List <Exception>(join.GetAllErrors()); if (errors.Count > 0) { string file = GenerateLogFileName("join-error-"); LogErrors(errors, file); base.WriteError("Errors were encountered during join operation"); retCode = CommandStatus.E_FAIL_JOIN_WITH_ERRORS; } else { retCode = CommandStatus.E_OK; } } } else if (TaskDefinitionHelper.IsSequentialProcess(_file)) { var def = (SequentialProcessDefinition)SequentialProcessDefinition.Serializer.Deserialize(File.OpenRead(_file)); var proc = new FdoSequentialProcess(def); proc.ProcessMessage += delegate(object sender, MessageEventArgs e) { base.WriteLine(e.Message); }; proc.Execute(); List <Exception> errors = new List <Exception>(proc.GetAllErrors()); if (errors.Count > 0) { string file = GenerateLogFileName("seq-process-"); LogErrors(errors, file); base.WriteError("Errors were encountered during sequential process"); } //Why E_OK? the user should check the log for the underlying return codes //of individual FdoUtil.exe invocations! retCode = CommandStatus.E_OK; } else { retCode = CommandStatus.E_FAIL_UNRECOGNISED_TASK_FORMAT; } return((int)retCode); }
/// <summary> /// Prepares the specified bulk copy definition (freshly deserialized) before the loading process begins /// </summary> /// <param name="def">The bulk copy definition.</param> protected override NameValueCollection Prepare(FdoBulkCopyTaskDefinition def) { return(new NameValueCollection()); }
/// <summary> /// Loads bulk copy options from deserialized xml /// </summary> /// <param name="def">The deserialized definition.</param> /// <param name="name">The name.</param> /// <param name="owner">if set to <c>true</c> [owner].</param> /// <returns></returns> public FdoBulkCopyOptions BulkCopyFromXml(FdoBulkCopyTaskDefinition def, ref string name, bool owner) { var nameMap = Prepare(def); // TODO/FIXME/HACK: // // The introduction of on-the-fly schema modifications before copying has introduced a // potential problem which will only occur when multiple copy tasks copy to the same target // feature class. // // What happens is because a target will be multiple sources copying to it, if we need // to create/update the target class definition, the current infrastructure will be confused // as to which source feature class to clone from. // // A possible solution is to return the first matching name. This will cause a "create" operation // to be queued. We then alter the pre-modification process so that subsequent "create" operations // becomes "update" operations instead. // // On second thought, the current infrastructure should be smart enought to prevent this, as // the UI only allows: // // 1. Mapping to an existing class // 2. Mapping to a class of the same name (create if necessary) // // It won't be possible to map to a class that doesn't exist, as the requirement for auto-create/update // is that the class name will be the same. // // ie. It won't be possible to create this mapping // // a. Foo -> Foo (create if necessary) // b. Bar -> Foo // // Rule 1 prevents mapping b) from happening // Rule 2 ensures that Foo will only ever be created once // // If multiple sources are mapped to the same target, I believe the bcp infrastructure should ensure // that target must already exist first. name = def.name; Dictionary <string, FdoConnection> connections = new Dictionary <string, FdoConnection>(); Dictionary <string, string> changeConnNames = new Dictionary <string, string>(); //TODO: Prepare() ensured that all connections have unique names that don't already exist //now in the effort to re-use existing connections, see which connection entries //already exist and can be renamed back to the old name foreach (FdoConnectionEntryElement entry in def.Connections) { string connName = entry.name; FdoConnection conn = CreateConnection(entry.provider, entry.ConnectionString, entry.configPath, ref connName); connections[connName] = conn; if (connName != entry.name) { changeConnNames[entry.name] = connName; } } foreach (string oldName in changeConnNames.Keys) { def.UpdateConnectionReferences(oldName, changeConnNames[oldName]); } //Compile the list of classes to be queried Dictionary <string, MultiSchemaQuery> queries = new Dictionary <string, MultiSchemaQuery>(); foreach (FdoCopyTaskElement task in def.CopyTasks) { MultiSchemaQuery src = null; MultiSchemaQuery dst = null; //Process source if (!queries.ContainsKey(task.Source.connection)) { src = queries[task.Source.connection] = new MultiSchemaQuery(task.Source.connection, SchemaOrigin.Source); } else { src = queries[task.Source.connection]; } var sq = src.TryGet(task.Source.schema); if (sq != null) { sq.AddClass(task.Source.@class); } else { sq = new SchemaQuery(task.Source.schema); sq.AddClass(task.Source.@class); src.Add(sq); } //Process target if (!queries.ContainsKey(task.Target.connection)) { dst = queries[task.Target.connection] = new MultiSchemaQuery(task.Target.connection, SchemaOrigin.Target); } else { dst = queries[task.Target.connection]; } var tq = dst.TryGet(task.Target.schema); if (tq != null) { tq.AddClass(task.Target.@class); } else { tq = new SchemaQuery(task.Target.schema); tq.AddClass(task.Target.@class); dst.Add(tq); } } List <TargetClassModificationItem> modifiers = new List <TargetClassModificationItem>(); using (var schemaCache = new FeatureSchemaCache()) { //Now populate the schema cache with source schemas foreach (string connName in queries.Keys) { if (connections.ContainsKey(connName)) { var mqry = queries[connName]; var conn = connections[connName]; FeatureSchemaCollection schemas = new FeatureSchemaCollection(null); using (var svc = conn.CreateFeatureService()) { foreach (var sq in mqry.SchemaQueries) { //Source schema queries are expected to fully check out so use original method if (mqry.Origin == SchemaOrigin.Source) { schemas.Add(svc.PartialDescribeSchema(sq.SchemaName, new List <string>(sq.ClassNames))); } } } if (schemas.Count > 0) { schemaCache.Add(connName, schemas); } } } //Now populate the schema cache with target schemas, taking note of any //classes that need to be created or updated. foreach (string connName in queries.Keys) { if (connections.ContainsKey(connName)) { var mqry = queries[connName]; var conn = connections[connName]; FeatureSchemaCollection schemas = new FeatureSchemaCollection(null); using (var svc = conn.CreateFeatureService()) { foreach (var sq in mqry.SchemaQueries) { //Source schema queries are expected to fully check out so use original method if (mqry.Origin == SchemaOrigin.Target) { //Because we may need to create new classes, the old method will break down on non-existent class names //So use the new method string[] notFound; var schema = svc.PartialDescribeSchema(sq.SchemaName, new List <string>(sq.ClassNames), out notFound); //if (notFound.Length > 0) //{ // //If we can't modify schemas we'll stop right here. This is caused by elements containing createIfNotExists = true // if (!conn.Capability.GetBooleanCapability(CapabilityType.FdoCapabilityType_SupportsSchemaModification)) // throw new NotSupportedException("The connection named " + connName + " does not support schema modification. Therefore, copy tasks and property/expression mappings cannot have createIfNotExists = true"); // //This cumbersome, but we need the parent schema name of the class that we will need to copy // string srcSchema = GetSourceSchemaForMapping(def, sq.SchemaName, notFound); // foreach (string className in notFound) // { // modifiers.Add(new CreateTargetClassFromSource(srcSchema, className)); // } //} schemas.Add(schema); } } } if (schemas.Count > 0) { schemaCache.Add(connName, schemas); } } } FdoBulkCopyOptions opts = new FdoBulkCopyOptions(connections, owner); foreach (FdoCopyTaskElement task in def.CopyTasks) { TargetClassModificationItem mod; FdoClassCopyOptions copt = FdoClassCopyOptions.FromElement(task, schemaCache, connections[task.Source.connection], connections[task.Target.connection], out mod); opts.AddClassCopyOption(copt); if (mod != null) { copt.PreCopyTargetModifier = mod; } //opts.AddClassModifier(mod); } return(opts); } }
/// <summary> /// Prepares the specified bulk copy definition (freshly deserialized) before the loading process begins /// </summary> /// <param name="def">The bulk copy definition.</param> /// <returns>A collection of [old name] - [new name] mappings</returns> protected abstract NameValueCollection Prepare(FdoBulkCopyTaskDefinition def);