/// <summary>
/// Loads bulk copy options from xml
/// </summary>
/// <param name="file">The file.</param>
/// <param name="name">The name.</param>
/// <param name="owner">if set to <c>true</c> [owner].</param>
/// <returns></returns>
public FdoBulkCopyOptions BulkCopyFromXml(string file, ref string name, bool owner)
{
FdoBulkCopyTaskDefinition def = null;
XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition));
def = (FdoBulkCopyTaskDefinition)ser.Deserialize(new StreamReader(file));
return(BulkCopyFromXml(def, ref name, owner));
}
/// <summary>
/// Saves this process to a file
/// </summary>
/// <param name="file">The file to save this process to</param>
/// <param name="name">The name of the process</param>
public override void Save(string file, string name)
{
FdoBulkCopyTaskDefinition def = new FdoBulkCopyTaskDefinition();
def.name = name;
List <FdoConnectionEntryElement> connList = new List <FdoConnectionEntryElement>();
List <FdoCopyTaskElement> copyTasks = new List <FdoCopyTaskElement>();
foreach (string connName in _options.ConnectionNames)
{
FdoConnection conn = _options.GetConnection(connName);
FdoConnectionEntryElement entry = new FdoConnectionEntryElement();
entry.name = connName;
entry.provider = conn.Provider;
entry.ConnectionString = conn.ConnectionString;
if (conn.HasConfiguration)
{
string path = Path.GetDirectoryName(file);
path = Path.Combine(path, entry.name + "_configuration.xml");
conn.SaveConfiguration(path);
entry.configPath = path;
}
connList.Add(entry);
}
foreach (FdoClassCopyOptions copt in _options.ClassCopyOptions)
{
copyTasks.Add(copt.ToElement());
}
def.Connections = connList.ToArray();
def.CopyTasks = copyTasks.ToArray();
using (StreamWriter writer = new StreamWriter(file, false))
{
XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition));
ser.Serialize(writer, def);
}
}
private static string GetSourceSchemaForMapping(FdoBulkCopyTaskDefinition def, string targetSchema, string[] classNames)
{
ISet <string> matches = new HashSet <string>();
List <string> classes = new List <string>(classNames);
foreach (var task in def.CopyTasks)
{
if (classes.Contains(task.Target.@class) && task.Target.schema.Equals(targetSchema))
{
matches.Add(task.Source.schema);
}
}
if (matches.Count > 1)
{
throw new TaskValidationException("The specified class names have various parent schema names");
}
if (matches.Count == 0)
{
throw new TaskValidationException("Could not determine parent schema name for the given class names");
}
return(new List <string>(matches)[0]); //The price to pay for targeting linq-less fx 2.0
}
private FdoBulkCopyTaskDefinition Save()
{
FdoBulkCopyTaskDefinition def = new FdoBulkCopyTaskDefinition();
def.name = txtName.Text;
List <FdoConnectionEntryElement> conns = new List <FdoConnectionEntryElement>();
foreach (DataGridViewRow row in grdConnections.Rows)
{
FdoConnectionEntryElement entry = new FdoConnectionEntryElement();
entry.name = row.Cells[0].Value.ToString();
entry.provider = row.Cells[1].Value.ToString();
entry.ConnectionString = row.Cells[3].Value.ToString();
conns.Add(entry);
}
List <FdoCopyTaskElement> tasks = new List <FdoCopyTaskElement>();
foreach (CopyTaskNodeDecorator dec in _tasks.Values)
{
FdoCopyTaskElement task = new FdoCopyTaskElement();
task.name = dec.Name;
task.createIfNotExists = dec.CreateIfNotExists;
task.Source = new FdoCopySourceElement();
task.Target = new FdoCopyTargetElement();
task.Options = new FdoCopyOptionsElement();
List <FdoPropertyMappingElement> pmaps = new List <FdoPropertyMappingElement>();
List <FdoExpressionMappingElement> emaps = new List <FdoExpressionMappingElement>();
//Source
task.Source.@class = dec.SourceClassName;
task.Source.connection = dec.SourceConnectionName;
task.Source.schema = dec.SourceSchemaName;
//Target
task.Target.@class = dec.TargetClassName;
task.Target.connection = dec.TargetConnectionName;
task.Target.schema = dec.TargetSchemaName;
//Options
task.Options.BatchSize = dec.Options.BatchSize.ToString();
task.Options.FlattenGeometries = dec.Options.Flatten;
task.Options.FlattenGeometriesSpecified = true;
task.Options.DeleteTarget = dec.Options.Delete;
task.Options.Filter = dec.Options.SourceFilter;
task.Options.ForceWKB = dec.Options.ForceWkb;
task.Options.ForceWKBSpecified = true;
//Property Mappings
NameValueCollection mappings = dec.PropertyMappings.GetPropertyMappings();
foreach (string srcProp in mappings.Keys)
{
string dstProp = mappings[srcProp];
FdoPropertyMappingElement p = new FdoPropertyMappingElement();
p.source = srcProp;
p.target = dstProp;
PropertyConversionNodeDecorator conv = dec.PropertyMappings.GetConversionRule(p.source);
p.nullOnFailedConversion = conv.NullOnFailedConversion;
p.truncate = conv.Truncate;
p.createIfNotExists = conv.CreateIfNotExists;
pmaps.Add(p);
}
foreach (string alias in dec.ExpressionMappings.GetAliases())
{
FdoExpressionMappingElement e = new FdoExpressionMappingElement();
e.alias = alias;
ExpressionMappingInfo exMap = dec.ExpressionMappings.GetMapping(alias);
e.Expression = exMap.Expression;
e.target = exMap.TargetProperty;
PropertyConversionNodeDecorator conv = dec.ExpressionMappings.GetConversionRule(e.alias);
e.nullOnFailedConversion = conv.NullOnFailedConversion;
e.truncate = conv.Truncate;
e.createIfNotExists = conv.CreateIfNotExists;
emaps.Add(e);
}
task.PropertyMappings = pmaps.ToArray();
task.ExpressionMappings = emaps.ToArray();
tasks.Add(task);
}
def.Connections = conns.ToArray();
def.CopyTasks = tasks.ToArray();
return(def);
}
public override int Execute()
{
CommandStatus retCode;
DefinitionLoader loader = new DefinitionLoader();
string name = null;
if (TaskDefinitionHelper.IsBulkCopy(_file))
{
FdoBulkCopyTaskDefinition def = null;
using (var sr = new StreamReader(_file))
{
XmlSerializer ser = new XmlSerializer(typeof(FdoBulkCopyTaskDefinition));
def = (FdoBulkCopyTaskDefinition)ser.Deserialize(sr);
}
if (def == null)
{
return((int)CommandStatus.E_FAIL_TASK_VALIDATION);
}
//If more than one task specified, load the default task and weed
//out unneeded elements.
if (_taskNames.Length > 0)
{
base.WriteLine("Certain tasks have been specified, only part of the bulk copy definition will execute");
var keepConnections = new Dictionary <string, FdoConnectionEntryElement>();
var keepTasks = new List <FdoCopyTaskElement>();
//Store needed tasks
foreach (var task in def.CopyTasks)
{
if (Array.IndexOf(_taskNames, task.name) >= 0)
{
keepTasks.Add(task);
}
}
//Store needed connections
foreach (var task in keepTasks)
{
foreach (var conn in def.Connections)
{
//Is referenced as source/target connection?
if (task.Source.connection == conn.name || task.Target.connection == conn.name)
{
if (!keepConnections.ContainsKey(conn.name))
{
keepConnections.Add(conn.name, conn);
}
}
}
}
if (keepTasks.Count != _taskNames.Length)
{
List <string> names = new List <string>();
foreach (var n in _taskNames)
{
bool found = false;
foreach (var task in keepTasks)
{
if (task.name == n)
{
found = true;
break;
}
}
if (!found)
{
names.Add(n);
}
}
base.WriteError("Could not find specified tasks in bulk copy definition: " + string.Join(",", names.ToArray()));
return((int)CommandStatus.E_FAIL_MISSING_BULK_COPY_TASKS);
}
//Now replace
def.Connections = new List <FdoConnectionEntryElement>(keepConnections.Values).ToArray();
def.CopyTasks = keepTasks.ToArray();
}
using (FdoBulkCopyOptions opts = loader.BulkCopyFromXml(def, ref name, true))
{
FdoBulkCopy copy = new FdoBulkCopy(opts);
copy.ProcessMessage += delegate(object sender, MessageEventArgs e)
{
base.WriteLine(e.Message);
};
copy.ProcessAborted += delegate(object sender, EventArgs e)
{
base.WriteLine("Bulk Copy Aborted");
};
copy.ProcessCompleted += delegate(object sender, EventArgs e)
{
base.WriteLine("Bulk Copy Completed");
};
copy.Execute();
List <Exception> errors = new List <Exception>(copy.GetAllErrors());
if (errors.Count > 0)
{
string file = GenerateLogFileName("bcp-error-");
LogErrors(errors, file);
base.WriteError("Errors were encountered during bulk copy.");
retCode = CommandStatus.E_FAIL_BULK_COPY_WITH_ERRORS;
}
else
{
retCode = CommandStatus.E_OK;
}
}
}
else if (TaskDefinitionHelper.IsJoin(_file))
{
if (_taskNames.Length > 0)
{
base.WriteError("Parameter -bcptask is not applicable for join tasks");
return((int)CommandStatus.E_FAIL_INVALID_ARGUMENTS);
}
using (FdoJoinOptions opts = loader.JoinFromXml(_file, ref name, true))
{
opts.Left.Connection.Open();
opts.Right.Connection.Open();
opts.Target.Connection.Open();
FdoJoin join = new FdoJoin(opts);
join.ProcessMessage += delegate(object sender, MessageEventArgs e)
{
base.WriteLine(e.Message);
};
join.Execute();
List <Exception> errors = new List <Exception>(join.GetAllErrors());
if (errors.Count > 0)
{
string file = GenerateLogFileName("join-error-");
LogErrors(errors, file);
base.WriteError("Errors were encountered during join operation");
retCode = CommandStatus.E_FAIL_JOIN_WITH_ERRORS;
}
else
{
retCode = CommandStatus.E_OK;
}
}
}
else if (TaskDefinitionHelper.IsSequentialProcess(_file))
{
var def = (SequentialProcessDefinition)SequentialProcessDefinition.Serializer.Deserialize(File.OpenRead(_file));
var proc = new FdoSequentialProcess(def);
proc.ProcessMessage += delegate(object sender, MessageEventArgs e)
{
base.WriteLine(e.Message);
};
proc.Execute();
List <Exception> errors = new List <Exception>(proc.GetAllErrors());
if (errors.Count > 0)
{
string file = GenerateLogFileName("seq-process-");
LogErrors(errors, file);
base.WriteError("Errors were encountered during sequential process");
}
//Why E_OK? the user should check the log for the underlying return codes
//of individual FdoUtil.exe invocations!
retCode = CommandStatus.E_OK;
}
else
{
retCode = CommandStatus.E_FAIL_UNRECOGNISED_TASK_FORMAT;
}
return((int)retCode);
}
/// <summary>
/// Prepares the specified bulk copy definition (freshly deserialized) before the loading process begins
/// </summary>
/// <param name="def">The bulk copy definition.</param>
protected override NameValueCollection Prepare(FdoBulkCopyTaskDefinition def)
{
return(new NameValueCollection());
}
/// <summary>
/// Loads bulk copy options from deserialized xml
/// </summary>
/// <param name="def">The deserialized definition.</param>
/// <param name="name">The name.</param>
/// <param name="owner">if set to <c>true</c> [owner].</param>
/// <returns></returns>
public FdoBulkCopyOptions BulkCopyFromXml(FdoBulkCopyTaskDefinition def, ref string name, bool owner)
{
var nameMap = Prepare(def);
// TODO/FIXME/HACK:
//
// The introduction of on-the-fly schema modifications before copying has introduced a
// potential problem which will only occur when multiple copy tasks copy to the same target
// feature class.
//
// What happens is because a target will be multiple sources copying to it, if we need
// to create/update the target class definition, the current infrastructure will be confused
// as to which source feature class to clone from.
//
// A possible solution is to return the first matching name. This will cause a "create" operation
// to be queued. We then alter the pre-modification process so that subsequent "create" operations
// becomes "update" operations instead.
//
// On second thought, the current infrastructure should be smart enought to prevent this, as
// the UI only allows:
//
// 1. Mapping to an existing class
// 2. Mapping to a class of the same name (create if necessary)
//
// It won't be possible to map to a class that doesn't exist, as the requirement for auto-create/update
// is that the class name will be the same.
//
// ie. It won't be possible to create this mapping
//
// a. Foo -> Foo (create if necessary)
// b. Bar -> Foo
//
// Rule 1 prevents mapping b) from happening
// Rule 2 ensures that Foo will only ever be created once
//
// If multiple sources are mapped to the same target, I believe the bcp infrastructure should ensure
// that target must already exist first.
name = def.name;
Dictionary <string, FdoConnection> connections = new Dictionary <string, FdoConnection>();
Dictionary <string, string> changeConnNames = new Dictionary <string, string>();
//TODO: Prepare() ensured that all connections have unique names that don't already exist
//now in the effort to re-use existing connections, see which connection entries
//already exist and can be renamed back to the old name
foreach (FdoConnectionEntryElement entry in def.Connections)
{
string connName = entry.name;
FdoConnection conn = CreateConnection(entry.provider, entry.ConnectionString, entry.configPath, ref connName);
connections[connName] = conn;
if (connName != entry.name)
{
changeConnNames[entry.name] = connName;
}
}
foreach (string oldName in changeConnNames.Keys)
{
def.UpdateConnectionReferences(oldName, changeConnNames[oldName]);
}
//Compile the list of classes to be queried
Dictionary <string, MultiSchemaQuery> queries = new Dictionary <string, MultiSchemaQuery>();
foreach (FdoCopyTaskElement task in def.CopyTasks)
{
MultiSchemaQuery src = null;
MultiSchemaQuery dst = null;
//Process source
if (!queries.ContainsKey(task.Source.connection))
{
src = queries[task.Source.connection] = new MultiSchemaQuery(task.Source.connection, SchemaOrigin.Source);
}
else
{
src = queries[task.Source.connection];
}
var sq = src.TryGet(task.Source.schema);
if (sq != null)
{
sq.AddClass(task.Source.@class);
}
else
{
sq = new SchemaQuery(task.Source.schema);
sq.AddClass(task.Source.@class);
src.Add(sq);
}
//Process target
if (!queries.ContainsKey(task.Target.connection))
{
dst = queries[task.Target.connection] = new MultiSchemaQuery(task.Target.connection, SchemaOrigin.Target);
}
else
{
dst = queries[task.Target.connection];
}
var tq = dst.TryGet(task.Target.schema);
if (tq != null)
{
tq.AddClass(task.Target.@class);
}
else
{
tq = new SchemaQuery(task.Target.schema);
tq.AddClass(task.Target.@class);
dst.Add(tq);
}
}
List <TargetClassModificationItem> modifiers = new List <TargetClassModificationItem>();
using (var schemaCache = new FeatureSchemaCache())
{
//Now populate the schema cache with source schemas
foreach (string connName in queries.Keys)
{
if (connections.ContainsKey(connName))
{
var mqry = queries[connName];
var conn = connections[connName];
FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);
using (var svc = conn.CreateFeatureService())
{
foreach (var sq in mqry.SchemaQueries)
{
//Source schema queries are expected to fully check out so use original method
if (mqry.Origin == SchemaOrigin.Source)
{
schemas.Add(svc.PartialDescribeSchema(sq.SchemaName, new List <string>(sq.ClassNames)));
}
}
}
if (schemas.Count > 0)
{
schemaCache.Add(connName, schemas);
}
}
}
//Now populate the schema cache with target schemas, taking note of any
//classes that need to be created or updated.
foreach (string connName in queries.Keys)
{
if (connections.ContainsKey(connName))
{
var mqry = queries[connName];
var conn = connections[connName];
FeatureSchemaCollection schemas = new FeatureSchemaCollection(null);
using (var svc = conn.CreateFeatureService())
{
foreach (var sq in mqry.SchemaQueries)
{
//Source schema queries are expected to fully check out so use original method
if (mqry.Origin == SchemaOrigin.Target)
{
//Because we may need to create new classes, the old method will break down on non-existent class names
//So use the new method
string[] notFound;
var schema = svc.PartialDescribeSchema(sq.SchemaName, new List <string>(sq.ClassNames), out notFound);
//if (notFound.Length > 0)
//{
// //If we can't modify schemas we'll stop right here. This is caused by elements containing createIfNotExists = true
// if (!conn.Capability.GetBooleanCapability(CapabilityType.FdoCapabilityType_SupportsSchemaModification))
// throw new NotSupportedException("The connection named " + connName + " does not support schema modification. Therefore, copy tasks and property/expression mappings cannot have createIfNotExists = true");
// //This cumbersome, but we need the parent schema name of the class that we will need to copy
// string srcSchema = GetSourceSchemaForMapping(def, sq.SchemaName, notFound);
// foreach (string className in notFound)
// {
// modifiers.Add(new CreateTargetClassFromSource(srcSchema, className));
// }
//}
schemas.Add(schema);
}
}
}
if (schemas.Count > 0)
{
schemaCache.Add(connName, schemas);
}
}
}
FdoBulkCopyOptions opts = new FdoBulkCopyOptions(connections, owner);
foreach (FdoCopyTaskElement task in def.CopyTasks)
{
TargetClassModificationItem mod;
FdoClassCopyOptions copt = FdoClassCopyOptions.FromElement(task, schemaCache, connections[task.Source.connection], connections[task.Target.connection], out mod);
opts.AddClassCopyOption(copt);
if (mod != null)
{
copt.PreCopyTargetModifier = mod;
}
//opts.AddClassModifier(mod);
}
return(opts);
}
}
/// <summary> /// Prepares the specified bulk copy definition (freshly deserialized) before the loading process begins /// </summary> /// <param name="def">The bulk copy definition.</param> /// <returns>A collection of [old name] - [new name] mappings</returns> protected abstract NameValueCollection Prepare(FdoBulkCopyTaskDefinition def);
