protected override Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var task1 = Task.Run(() => ExecuteSqlCmd(File
.ReadAllText($@"SQL\{GetType().Name}Way.sql")
.Replace("[OSM]", Connection.InitialCatalog)));
var task2 = Task.Run(() => ExecuteSqlCmd(File
.ReadAllText($@"SQL\{GetType().Name}Node.sql")
.Replace("[OSM]", Connection.InitialCatalog)));
return(Task.WhenAll(new[] { task1, task2 }));
}
protected override void DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
_database = Connection.InitialCatalog;
_osmFile = osmFile;
Connection.InitialCatalog = string.Empty;
CreateDatabase();
CreateTables();
Connection.InitialCatalog = _database;
}
public void TestArrayDescriptor()
{
var attributeRegistry = new AttributeRegistry();
var factory = new TypeDescriptorFactory(attributeRegistry);
var descriptor = new ArrayDescriptor(factory, typeof(int[]), false, new DefaultNamingConvention());
descriptor.Initialize(new DefaultKeyComparer());
Assert.Equal(0, descriptor.Count);
Assert.Equal(typeof(int), descriptor.ElementType);
}
protected override Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var reader = osmFile.EndsWith(".pbf")
? (IOsmReader) new PbfOsmReader()
: (IOsmReader) new XmlOsmReader();
ExecuteSqlCmd("TRUNCATE TABLE [dbo].[tRelationCreation]");
ExecuteSqlCmd("TRUNCATE TABLE [dbo].[tRelationTag]");
var dRelationCreation = new DataTable {
MinimumCapacity = MaxRowCountInMemory
};
dRelationCreation.TableName = "tRelationCreation";
dRelationCreation.Columns.Add("RelationId", typeof(long));
dRelationCreation.Columns.Add("Ref", typeof(long));
dRelationCreation.Columns.Add("Type");
dRelationCreation.Columns.Add("Role");
dRelationCreation.Columns.Add("Sort");
var dRelationTags = new DataTable {
MinimumCapacity = MaxRowCountInMemory
};
dRelationTags.TableName = "tRelationTag";
dRelationTags.Columns.Add("RelationId", typeof(long));
dRelationTags.Columns.Add("Typ");
dRelationTags.Columns.Add("Info");
var insertingTask = Task.Factory.StartNew(() => StartInserting());
foreach (var relation in reader.ReadRelations(osmFile, attributeRegistry))
{
_countOfInsertedRelations++;
var sort = 0;
foreach (var member in relation.Members)
{
dRelationCreation = AddToCollection(dRelationCreation, relation.RelationId, member.Ref, member.Type, member.Role, sort++);
}
foreach (var tag in relation.Tags)
{
dRelationTags = AddToCollection(dRelationTags, relation.RelationId, tag.Typ, tag.Value);
}
}
DataTableCollection.Add(dRelationCreation);
DataTableCollection.Add(dRelationTags);
DataTableCollection.CompleteAdding();
Trace.WriteLine(string.Format("Inserted {0} relations", _countOfInsertedRelations));
return(insertingTask.Result);
}
public void TestPrimitiveDescriptor()
{
var attributeRegistry = new AttributeRegistry();
var descriptor = new PrimitiveDescriptor(attributeRegistry, typeof(int), new DefaultNamingConvention());
Assert.AreEqual(0, descriptor.Count);
Assert.True(PrimitiveDescriptor.IsPrimitive(typeof(MyEnum)));
Assert.True(PrimitiveDescriptor.IsPrimitive(typeof(object)));
Assert.True(PrimitiveDescriptor.IsPrimitive(typeof(DateTime)));
Assert.True(PrimitiveDescriptor.IsPrimitive(typeof(TimeSpan)));
Assert.False(PrimitiveDescriptor.IsPrimitive(typeof(IList)));
}
protected override async Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
await Task.Run(() =>
{
_database = Connection.InitialCatalog;
_osmFile = osmFile;
Connection.InitialCatalog = string.Empty;
CreateTables();
Connection.InitialCatalog = _database;
});
}
protected override async Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var res = await QuerySqlCmd <string>("SELECT @@VERSION;");
var createExtensions = App.GetResourceFileText("osm2mssql.Importer.SQL.CreateDbExtension.sql");
var file = Directory.GetCurrentDirectory() + @"\osm2mssql.OsmDb.dll";
var buffer = File.ReadAllBytes(file);
var data = "0x" + string.Join("", buffer.Select(x => x.ToString("X2")));
createExtensions = createExtensions.Replace("[OSM]", Connection.InitialCatalog);
createExtensions = createExtensions.Replace("[DllExtension]", data);
ExecuteSqlCmd(createExtensions);
}
public void TestObjectWithCustomNamingConvention()
{
var attributeRegistry = new AttributeRegistry();
var factory = new TypeDescriptorFactory(attributeRegistry);
var descriptor = new ObjectDescriptor(factory, typeof(TestObjectNamingConvention), false, new FlatNamingConvention());
descriptor.Initialize(new DefaultKeyComparer());
// Check names and their orders
Assert.AreEqual(descriptor.Members.Select(memberDescriptor => memberDescriptor.Name), new[]
{
"myname",
"name",
"this_is_camel_name"
});
}
protected override Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
ExecuteSqlCmd("TRUNCATE TABLE [Node]");
ExecuteSqlCmd("TRUNCATE TABLE [NodeTag]");
var loadingNodeTable = new DataTable();
loadingNodeTable.TableName = "Node";
loadingNodeTable.MinimumCapacity = MaxRowCountInMemory;
loadingNodeTable.Columns.Add("NodeId", typeof(long));
loadingNodeTable.Columns.Add("location", typeof(SqlGeography));
loadingNodeTable.Columns.Add("Latitude", typeof(double));
loadingNodeTable.Columns.Add("Longitude", typeof(double));
var dNodeTags = new DataTable();
dNodeTags.TableName = "NodeTag";
dNodeTags.MinimumCapacity = MaxRowCountInMemory;
dNodeTags.Columns.Add("NodeId", typeof(long));
dNodeTags.Columns.Add("Typ");
dNodeTags.Columns.Add("Info");
IOsmReader reader = osmFile.EndsWith(".pbf") ? (IOsmReader) new PbfOsmReader() : (IOsmReader) new XmlOsmReader();
var insertingTask = Task.Factory.StartNew(() => StartInserting());
foreach (var node in reader.ReadNodes(osmFile, attributeRegistry))
{
_countOfInsertedNodes++;
loadingNodeTable = AddToCollection(loadingNodeTable, node.NodeId, node.ToSqlGeographyPoint(), node.Latitude, node.Longitude);
foreach (var tag in node.Tags)
{
dNodeTags = AddToCollection(dNodeTags, node.NodeId, tag.Typ, tag.Value);
}
}
DataTableCollection.Add(loadingNodeTable);
DataTableCollection.Add(dNodeTags);
DataTableCollection.CompleteAdding();
Trace.WriteLine(string.Format("Inserted {0} nodes", _countOfInsertedNodes));
return(insertingTask.Result);
}
private void RunAllTasksAsync(SqlConnectionStringBuilder connectionStringBuilder, string fileName)
{
foreach (var task in Tasks)
{
task.Result = TaskResult.Pending;
}
var attributeRegistry = new AttributeRegistry();
var watch = Stopwatch.StartNew();
RunTasksSynchron(Tasks.Where(d => d.IsEnabled && d.Type == TaskType.InitializeTask), connectionStringBuilder, fileName, attributeRegistry);
RunTasksParallel(Tasks.Where(d => d.IsEnabled && d.Type == TaskType.ParallelTask), connectionStringBuilder, fileName, attributeRegistry);
RunTasksParallel(Tasks.Where(d => d.IsEnabled && d.Type == TaskType.ParallelFinishTask), connectionStringBuilder, fileName, attributeRegistry);
RunTasksSynchron(Tasks.Where(d => d.IsEnabled && d.Type == TaskType.FinishTask), connectionStringBuilder, fileName, attributeRegistry);
var processSummary = string.Format("Totalduration: {0}", watch.Elapsed);
Trace.WriteLine(processSummary);
}
protected override void DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var sql = @"TRUNCATE TABLE tRelation";
ExecuteSqlCmd(sql);
sql = @"INSERT INTO tRelation(id, geo, role)
SELECT rel.RelationId, dbo.GeographyUnion(x.geo) as geo, rel.role FROM tRelationCreation rel LEFT JOIN
(SELECT Id, Location as geo, (SELECT id FROM tMemberType Where Name = 'node') as [Type] FROM tNode
UNION ALL
SELECT Id, Line as geo, (SELECT id FROM tMemberType Where Name = 'way') as [Type] FROM tWay) x
on rel.ref = x.Id and rel.type = x.Type
group by RelationId, rel.role";
ExecuteSqlCmd(sql);
sql = @"UPDATE tRelation set geo = dbo.ConvertToPolygon(geo)
where tRelation.id IN (SELECT RelationId FROM tRelationTag tag where tag.Info = 'multipolygon')";
ExecuteSqlCmd(sql);
}
public void TestDictionaryDescriptor()
{
var attributeRegistry = new AttributeRegistry();
var descriptor = new DictionaryDescriptor(attributeRegistry, typeof(Dictionary <int, string>), false);
descriptor.Initialize();
Assert.AreEqual(0, descriptor.Count);
Assert.True(descriptor.IsPureDictionary);
Assert.AreEqual(typeof(int), descriptor.KeyType);
Assert.AreEqual(typeof(string), descriptor.ValueType);
descriptor = new DictionaryDescriptor(attributeRegistry, typeof(NonPureDictionary), false);
descriptor.Initialize();
Assert.AreEqual(1, descriptor.Count);
Assert.False(descriptor.IsPureDictionary);
Assert.AreEqual(typeof(float), descriptor.KeyType);
Assert.AreEqual(typeof(object), descriptor.ValueType);
}
/// <summary>
/// Initializes a new instance of the <see cref="ObjectDescriptor" /> class.
/// </summary>
/// <param name="attributeRegistry">The attribute registry.</param>
/// <param name="type">The type.</param>
/// <param name="emitDefaultValues"></param>
/// <exception cref="System.ArgumentNullException">type</exception>
/// <exception cref="YamlException">Failed to get ObjectDescriptor for type [{0}]. The member [{1}] cannot be registered as a member with the same name is already registered [{2}].DoFormat(type.FullName, member, existingMember)</exception>
public ObjectDescriptor(IAttributeRegistry attributeRegistry, Type type, bool emitDefaultValues)
{
if (attributeRegistry == null)
{
throw new ArgumentNullException("attributeRegistry");
}
if (type == null)
{
throw new ArgumentNullException("type");
}
this.emitDefaultValues = emitDefaultValues;
this.AttributeRegistry = attributeRegistry;
this.type = type;
var styleAttribute = AttributeRegistry.GetAttribute <YamlStyleAttribute>(type);
this.style = styleAttribute != null ? styleAttribute.Style : YamlStyle.Any;
this.IsCompilerGenerated = AttributeRegistry.GetAttribute <CompilerGeneratedAttribute>(type) != null;
}
protected async override void DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var res = await QuerySqlCmd <string>("SELECT @@VERSION;");
bool is2008Server = res.Contains("Server 2008");
Task task1 = null, task2 = null;
if (is2008Server)
{
task1 = Task.Factory.StartNew(() => ExecuteSqlCmd("CREATE SPATIAL INDEX idx ON tWay(line)"));
task2 = Task.Factory.StartNew(() => ExecuteSqlCmd("CREATE SPATIAL INDEX idx ON tNode(location)"));
}
else
{
task1 = Task.Factory.StartNew(() => ExecuteSqlCmd("CREATE SPATIAL INDEX idx ON tWay(line) USING GEOGRAPHY_AUTO_GRID"));
task2 = Task.Factory.StartNew(() => ExecuteSqlCmd("CREATE SPATIAL INDEX idx ON tNode(location) USING GEOGRAPHY_AUTO_GRID"));
}
Task.WaitAll(new[] { task1, task2 });
}
public void Reset()
{
modelType = Builder.TypeDecl();
modelType.Namespace = ModelNamespace;
modelType.Owner = null;
modelType.BaseType = null;
modelType.Visibility = TypeVisibility.Public;
modelMethod = Builder.MethodDecl(MethodVisibility.Public, "Model", typeof(void), modelType);
IBlockStatement body = Builder.BlockStmt();
modelMethod.Body = body;
blockStack = new Stack <IList <IStatement> >();
blockStack.Push(body.Statements);
modelType.Methods.Add(modelMethod);
Attributes = new AttributeRegistry <object, ICompilerAttribute>(true);
searched.Clear();
observedVars.Clear();
variablesToInfer.Clear();
constants.Clear();
ModelExpressions = null;
}
/// <summary>
/// Method for starting the work of this Task
/// </summary>
/// <param name="connection"></param>
public void RunTask(SqlConnectionStringBuilder connection, string osmFile, AttributeRegistry attributeRegistry)
{
_startTime = DateTime.Now;
try
{
Result = TaskResult.InProgress;
Connection = connection;
DoTaskWork(osmFile, attributeRegistry);
var logLine = string.Format("Finished Task {0} - Duration: {1}", Name, Duration);
Trace.WriteLine(logLine);
Result = TaskResult.Successful;
}
catch (Exception ex)
{
Trace.WriteLine(ex);
Result = TaskResult.Error;
LastError = ex;
throw;
}
}
protected override Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var runningTasks = new List <Task>();
var tagTypes = attributeRegistry.GetAttributeValues(OsmAttribute.TagType);
var memberRoles = attributeRegistry.GetAttributeValues(OsmAttribute.MemberRole);
var memberTypes = attributeRegistry.GetAttributeValues(OsmAttribute.MemberType);
var tagTypeDataTable = new DataTable("TagType");
tagTypeDataTable.Columns.Add("Typ", typeof(int));
tagTypeDataTable.Columns.Add("Name", typeof(string));
foreach (var tagType in tagTypes)
{
tagTypeDataTable.Rows.Add(tagType.Key, tagType.Value);
}
runningTasks.Add(WriteToDbAsync(tagTypeDataTable.TableName, tagTypeDataTable));
var memberRoleDataTable = new DataTable("MemberRole");
memberRoleDataTable.Columns.Add("id", typeof(int));
memberRoleDataTable.Columns.Add("name", typeof(string));
foreach (var tagType in memberRoles)
{
memberRoleDataTable.Rows.Add(tagType.Key, tagType.Value);
}
runningTasks.Add(WriteToDbAsync(memberRoleDataTable.TableName, memberRoleDataTable));
var memberTypesDataTable = new DataTable("MemberType");
memberTypesDataTable.Columns.Add("id", typeof(int));
memberTypesDataTable.Columns.Add("name", typeof(string));
foreach (var tagType in memberTypes)
{
memberTypesDataTable.Rows.Add(tagType.Key, tagType.Value);
}
runningTasks.Add(WriteToDbAsync(memberTypesDataTable.TableName, memberTypesDataTable));
return(Task.WhenAll(runningTasks.ToArray()));
}
protected override void DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
var tagTypes = attributeRegistry.GetAttributeValues(OsmAttribute.TagType);
var tagTypeDataTable = new DataTable("tTagType");
tagTypeDataTable.Columns.Add("Typ", typeof(int));
tagTypeDataTable.Columns.Add("Name", typeof(string));
foreach (var tagType in tagTypes)
{
tagTypeDataTable.Rows.Add(tagType.Key, tagType.Value);
}
WriteToDb(tagTypeDataTable.TableName, tagTypeDataTable);
var memberRoles = attributeRegistry.GetAttributeValues(OsmAttribute.MemberRole);
var memberRoleDataTable = new DataTable("tMemberRole");
memberRoleDataTable.Columns.Add("id", typeof(int));
memberRoleDataTable.Columns.Add("name", typeof(string));
foreach (var tagType in memberRoles)
{
memberRoleDataTable.Rows.Add(tagType.Key, tagType.Value);
}
WriteToDb(memberRoleDataTable.TableName, memberRoleDataTable);
var memberTypes = attributeRegistry.GetAttributeValues(OsmAttribute.MemberType);
var memberTypesDataTable = new DataTable("tMemberType");
memberTypesDataTable.Columns.Add("id", typeof(int));
memberTypesDataTable.Columns.Add("name", typeof(string));
foreach (var tagType in memberTypes)
{
memberTypesDataTable.Rows.Add(tagType.Key, tagType.Value);
}
WriteToDb(memberTypesDataTable.TableName, memberTypesDataTable);
WaitUntilFinished();
}
public void TestDictionaryDescriptor()
{
var attributeRegistry = new AttributeRegistry();
var factory = new TypeDescriptorFactory(attributeRegistry);
var descriptor = new DictionaryDescriptor(factory, typeof(Dictionary <int, string>), false,
new DefaultNamingConvention());
descriptor.Initialize(new DefaultKeyComparer());
Assert.AreEqual(0, descriptor.Count);
Assert.True(descriptor.IsPureDictionary);
Assert.AreEqual(typeof(int), descriptor.KeyType);
Assert.AreEqual(typeof(string), descriptor.ValueType);
descriptor = new DictionaryDescriptor(factory, typeof(NonPureDictionary), false,
new DefaultNamingConvention());
descriptor.Initialize(new DefaultKeyComparer());
Assert.AreEqual(1, descriptor.Count);
Assert.False(descriptor.IsPureDictionary);
Assert.AreEqual(typeof(float), descriptor.KeyType);
Assert.AreEqual(typeof(object), descriptor.ValueType);
}
/// <summary>
/// Initializes a new instance of the <see cref="ObjectDescriptor" /> class.
/// </summary>
/// <param name="attributeRegistry">The attribute registry.</param>
/// <param name="type">The type.</param>
/// <param name="emitDefaultValues">if set to <c>true</c> [emit default values].</param>
/// <param name="ignoreGetters">If set to <c>true</c>, the properties without setters will be ignored.</param>
/// <param name="namingConvention">The naming convention.</param>
/// <exception cref="System.ArgumentNullException">type</exception>
/// <exception cref="YamlException">type</exception>
public ObjectDescriptor(IAttributeRegistry attributeRegistry, Type type, bool emitDefaultValues, bool ignoreGetters, IMemberNamingConvention namingConvention)
{
if (attributeRegistry == null)
{
throw new ArgumentNullException("attributeRegistry");
}
if (type == null)
{
throw new ArgumentNullException("type");
}
if (namingConvention == null)
{
throw new ArgumentNullException("namingConvention");
}
this.memberNamingConvention = namingConvention;
this.emitDefaultValues = emitDefaultValues;
this.ignoreGetters = ignoreGetters;
this.AttributeRegistry = attributeRegistry;
this.type = type;
attributes = AttributeRegistry.GetAttributes(type.GetTypeInfo());
this.style = YamlStyle.Any;
foreach (var attribute in attributes)
{
var styleAttribute = attribute as YamlStyleAttribute;
if (styleAttribute != null)
{
style = styleAttribute.Style;
continue;
}
if (attribute is CompilerGeneratedAttribute)
{
this.IsCompilerGenerated = true;
}
}
}
/// <summary>
/// Allows the algorithm to modify the attributes on a factor. For example, in Gibbs sampling
/// different message types are passed depending on the context. This is signalled to the MessageTransform
/// by attaching a MessagePath attribute to the method invoke expression for the factor.
/// If the factor is a 'variable' pseudo-factor (UsesEqualsDef) then all incoming variables are
/// Distributions. Otherwise, incoming messages will depend on the grouping
/// </summary>
/// <param name="factorExpression">The factor expression</param>
/// <param name="factorAttributes">Attribute registry</param>
public override void ModifyFactorAttributes(IExpression factorExpression, AttributeRegistry <object, ICompilerAttribute> factorAttributes)
{
IList <MessagePathAttribute> mpas = factorAttributes.GetAll <MessagePathAttribute>(factorExpression);
bool isVariable = factorAttributes.Has <IsVariableFactor>(factorExpression);
if (isVariable)
{
return;
}
// Process any Message Path attributes that may have been set by the Group transform
foreach (MessagePathAttribute mpa in mpas)
{
if (mpa.FromDistance >= mpa.ToDistance)
{
mpa.Path = "Distribution";
}
else
{
mpa.Path = "CurrentSample";
}
}
}
protected virtual List <IMemberDescriptor> PrepareMembers()
{
var bindingFlags = BindingFlags.Instance | BindingFlags.Public;
if (Category == DescriptorCategory.Object)
{
bindingFlags |= BindingFlags.NonPublic;
}
// Add all public properties with a readable get method
var memberList = (from propertyInfo in type.GetProperties(bindingFlags)
where
propertyInfo.CanRead && propertyInfo.GetIndexParameters().Length == 0
select new PropertyDescriptor(propertyInfo, NamingConvention.Comparer)
into member
where PrepareMember(member)
select member).Cast <IMemberDescriptor>().ToList();
// Add all public fields
foreach (var item in (from fieldInfo in type.GetFields(bindingFlags)
select new FieldDescriptor(fieldInfo, NamingConvention.Comparer)
into member
where PrepareMember(member)
select member))
{
memberList.Add(item);
}
// Allow to add dynamic members per type
if (AttributeRegistry.PrepareMembersCallback != null)
{
AttributeRegistry.PrepareMembersCallback(this, memberList);
}
return(memberList);
}
protected virtual bool PrepareMember(MemberDescriptorBase member)
{
var memberType = member.Type;
// Remove all SyncRoot from members
if (member is PropertyDescriptor && member.OriginalName == "SyncRoot" &&
(member.DeclaringType.Namespace ?? string.Empty).StartsWith(SystemCollectionsNamespace))
{
return(false);
}
// Process all attributes just once instead of getting them one by one
var attributes = AttributeRegistry.GetAttributes(member.MemberInfo);
YamlStyleAttribute styleAttribute = null;
YamlMemberAttribute memberAttribute = null;
DefaultValueAttribute defaultValueAttribute = null;
foreach (var attribute in attributes)
{
// Member is not displayed if there is a YamlIgnore attribute on it
if (attribute is YamlIgnoreAttribute)
{
return(false);
}
if (attribute is YamlMemberAttribute)
{
memberAttribute = (YamlMemberAttribute)attribute;
continue;
}
if (attribute is DefaultValueAttribute)
{
defaultValueAttribute = (DefaultValueAttribute)attribute;
continue;
}
if (attribute is YamlStyleAttribute)
{
styleAttribute = (YamlStyleAttribute)attribute;
continue;
}
var yamlRemap = attribute as YamlRemapAttribute;
if (yamlRemap != null)
{
if (member.AlternativeNames == null)
{
member.AlternativeNames = new List <string>();
}
if (!string.IsNullOrEmpty(yamlRemap.Name))
{
member.AlternativeNames.Add(yamlRemap.Name);
}
}
}
// If the member has a set, this is a conventional assign method
if (member.HasSet)
{
member.SerializeMemberMode = SerializeMemberMode.Content;
}
else
{
// Else we cannot only assign its content if it is a class
member.SerializeMemberMode = (memberType != typeof(string) && memberType.GetTypeInfo().IsClass) || memberType.GetTypeInfo().IsInterface || type.IsAnonymous() ? SerializeMemberMode.Content : SerializeMemberMode.Never;
}
// If it's a private member, check it has a YamlMemberAttribute on it
if (!member.IsPublic)
{
if (memberAttribute == null)
{
return(false);
}
}
// Gets the style
member.Style = styleAttribute != null ? styleAttribute.Style : YamlStyle.Any;
member.Mask = 1;
// Handle member attribute
if (memberAttribute != null)
{
member.Mask = memberAttribute.Mask;
if (!member.HasSet)
{
if (memberAttribute.SerializeMethod == SerializeMemberMode.Assign ||
(memberType.GetTypeInfo().IsValueType&& member.SerializeMemberMode == SerializeMemberMode.Content))
{
throw new ArgumentException("{0} {1} is not writeable by {2}.".DoFormat(memberType.FullName, member.OriginalName, memberAttribute.SerializeMethod.ToString()));
}
}
if (memberAttribute.SerializeMethod != SerializeMemberMode.Default)
{
member.SerializeMemberMode = memberAttribute.SerializeMethod;
}
member.Order = memberAttribute.Order;
}
if (member.SerializeMemberMode == SerializeMemberMode.Binary)
{
if (!memberType.IsArray)
{
throw new InvalidOperationException("{0} {1} of {2} is not an array. Can not be serialized as binary."
.DoFormat(memberType.FullName, member.OriginalName, type.FullName));
}
if (!memberType.GetElementType().IsPureValueType())
{
throw new InvalidOperationException("{0} is not a pure ValueType. {1} {2} of {3} can not serialize as binary.".DoFormat(memberType.GetElementType(), memberType.FullName, member.OriginalName, type.FullName));
}
}
// If this member cannot be serialized, remove it from the list
if (member.SerializeMemberMode == SerializeMemberMode.Never)
{
return(false);
}
// ShouldSerialize
// YamlSerializeAttribute(Never) => false
// ShouldSerializeSomeProperty => call it
// DefaultValueAttribute(default) => compare to it
// otherwise => true
var shouldSerialize = type.GetMethod("ShouldSerialize" + member.OriginalName, BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
if (shouldSerialize != null && shouldSerialize.ReturnType == typeof(bool) && member.ShouldSerialize == null)
{
member.ShouldSerialize = obj => (bool)shouldSerialize.Invoke(obj, EmptyObjectArray);
}
if (defaultValueAttribute != null && member.ShouldSerialize == null && !emitDefaultValues)
{
object defaultValue = defaultValueAttribute.Value;
Type defaultType = defaultValue == null ? null : defaultValue.GetType();
if (defaultType.IsNumeric() && defaultType != memberType)
{
defaultValue = memberType.CastToNumericType(defaultValue);
}
member.ShouldSerialize = obj => !TypeExtensions.AreEqual(defaultValue, member.Get(obj));
}
if (member.ShouldSerialize == null)
{
member.ShouldSerialize = ShouldSerializeDefault;
}
if (memberAttribute != null && !string.IsNullOrEmpty(memberAttribute.Name))
{
member.Name = memberAttribute.Name;
}
else
{
member.Name = NamingConvention.Convert(member.OriginalName);
}
return(true);
}
public StructField CreateField(StructDef structDef, string name, AttributeRegistry registry)
{
FieldAlias alias;
if (_aliasRegistry.TryGetValue(name, out alias))
{
StructField baseField = CreateField(structDef, alias.BaseName, registry);
foreach(StructParser.Attribute attr in alias.Attrs)
registry.SetFieldAttribute(baseField, attr.Key, attr.Value, attr.Position);
return baseField;
}
switch(name)
{
case "str": return new StrField(structDef, false, false);
case "cstr": return new StrField(structDef, false, true);
case "wstr": return new StrField(structDef, true, false);
case "child": return new ChildField(structDef, false);
case "sibling": return new ChildField(structDef, true);
case "seek": return new SeekField(structDef, false);
case "skip": return new SeekField(structDef, true);
case "rewind": return new RewindField(structDef);
case "repeat": return new RepeatField(structDef);
case "if": return new IfField(structDef);
case "elif": return new ElseIfField(structDef);
case "else": return new ElseField(structDef);
case "while": return new WhileField(structDef);
case "include": return new IncludeField(structDef);
case "assert": return new AssertField(structDef);
case "bitfield": return new BitfieldField(structDef);
case "nodename": return new NodenameField(structDef);
case "switch": return new SwitchField(structDef);
case "case": return new CaseField(structDef, false);
case "default": return new CaseField(structDef, true);
case "unixtime": return new UnixTimeField(structDef);
case "dosdatetime": return new DosDateTimeField(structDef);
case "global": return new GlobalField(structDef);
case "local": return new CalcField(structDef, true);
case "calc": return new CalcField(structDef, false);
case "align": return new AlignField(structDef);
case "blob": return new BlobField(structDef);
case "image": return new ImageField(structDef);
case "float": return new FloatField(structDef);
case "break": return new BreakField(structDef);
case "i": return new IntField(structDef, 0, false, false);
case "u": return new IntField(structDef, 0, true, false);
case "x": return new IntField(structDef, 0, true, true);
case "enum": return new EnumField(structDef, 0);
case "message": return new MessageField(structDef, false);
case "error": return new MessageField(structDef, true);
}
int size;
if (name.EndsWith("8"))
{
size = 1;
name = name.Substring(0, name.Length - 1);
}
else if (name.EndsWith("16") || name.EndsWith("32") || name.EndsWith("64"))
{
size = name.EndsWith("16") ? 2 : name.EndsWith("32") ? 4 : 8;
name = name.Substring(0, name.Length - 2);
}
else
throw new Exception("Unknown field type " + name);
switch(name)
{
case "i": return new IntField(structDef, size, false, false);
case "u": return new IntField(structDef, size, true, false);
case "x": return new IntField(structDef, size, true, true);
case "enum": return new EnumField(structDef, size);
case "set": return new SetField(structDef, size);
case "bits": return new BitsField(structDef, size);
}
throw new Exception("Unknown field type " + name);
}
protected override void DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
if (string.IsNullOrEmpty(osmFile))
{
return;
}
if (!File.Exists(osmFile))
{
return;
}
var watch = Stopwatch.StartNew();
ExecuteSqlCmd("TRUNCATE TABLE tWayCreation");
var dtWays = new DataTable();
dtWays.Columns.Add("wayId", typeof(long));
dtWays.Columns.Add("nodeId", typeof(long));
dtWays.Columns.Add("sort");
dtWays.MinimumCapacity = MaxRowCountInMemory;
var dtWayTags = new DataTable();
dtWayTags.MinimumCapacity = MaxRowCountInMemory;
dtWayTags.Columns.Add("WayId", typeof(long));
dtWayTags.Columns.Add("Typ", typeof(int));
dtWayTags.Columns.Add("Info", typeof(string));
var reader = osmFile.EndsWith(".pbf") ?
(IOsmReader) new PbfOsmReader() :
(IOsmReader) new XmlOsmReader();
foreach (var way in reader.ReadWays(osmFile, attributeRegistry))
{
if (!_timeOffset.HasValue)
{
watch.Stop();
_timeOffset = watch.Elapsed.TotalSeconds;
}
_countOfInsertedWays++;
var sort = 0;
foreach (var node in way.NodeRefs)
{
dtWays.Rows.Add(way.WayId, node, sort++);
}
var allText = string.Empty;
foreach (var tag in way.Tags)
{
allText += tag.Value + " ";
dtWayTags.Rows.Add(way.WayId, tag.Typ, tag.Value);
}
if (dtWays.Rows.Count >= MaxRowCountInMemory)
{
WriteToDb("tWayCreation", dtWays);
}
if (dtWayTags.Rows.Count >= MaxRowCountInMemory)
{
WriteToDb("tWayTag", dtWayTags);
}
}
WriteToDb("tWayCreation", dtWays);
WriteToDb("tWayTag", dtWayTags);
WaitUntilFinished();
Trace.WriteLine(string.Format("Inserted {0} ways", _countOfInsertedWays));
}
protected abstract Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry);
public StructField CreateField(StructDef structDef, string name, AttributeRegistry registry)
{
FieldAlias alias;
if (_aliasRegistry.TryGetValue(name, out alias))
{
StructField baseField = CreateField(structDef, alias.BaseName, registry);
foreach (StructParser.Attribute attr in alias.Attrs)
{
registry.SetFieldAttribute(baseField, attr.Key, attr.Value, attr.Position);
}
return(baseField);
}
switch (name)
{
case "str": return(new StrField(structDef, false, false));
case "cstr": return(new StrField(structDef, false, true));
case "wstr": return(new StrField(structDef, true, false));
case "child": return(new ChildField(structDef, false));
case "sibling": return(new ChildField(structDef, true));
case "seek": return(new SeekField(structDef, false));
case "skip": return(new SeekField(structDef, true));
case "rewind": return(new RewindField(structDef));
case "repeat": return(new RepeatField(structDef));
case "if": return(new IfField(structDef));
case "elif": return(new ElseIfField(structDef));
case "else": return(new ElseField(structDef));
case "while": return(new WhileField(structDef));
case "include": return(new IncludeField(structDef));
case "assert": return(new AssertField(structDef));
case "bitfield": return(new BitfieldField(structDef));
case "nodename": return(new NodenameField(structDef));
case "switch": return(new SwitchField(structDef));
case "case": return(new CaseField(structDef, false));
case "default": return(new CaseField(structDef, true));
case "unixtime": return(new UnixTimeField(structDef));
case "dosdatetime": return(new DosDateTimeField(structDef));
case "global": return(new GlobalField(structDef));
case "local": return(new CalcField(structDef, true));
case "calc": return(new CalcField(structDef, false));
case "align": return(new AlignField(structDef));
case "blob": return(new BlobField(structDef));
case "image": return(new ImageField(structDef));
case "float": return(new FloatField(structDef));
case "break": return(new BreakField(structDef));
case "i": return(new IntField(structDef, 0, false, false));
case "u": return(new IntField(structDef, 0, true, false));
case "x": return(new IntField(structDef, 0, true, true));
case "enum": return(new EnumField(structDef, 0));
case "message": return(new MessageField(structDef, false));
case "error": return(new MessageField(structDef, true));
}
int size;
if (name.EndsWith("8"))
{
size = 1;
name = name.Substring(0, name.Length - 1);
}
else if (name.EndsWith("16") || name.EndsWith("32") || name.EndsWith("64"))
{
size = name.EndsWith("16") ? 2 : name.EndsWith("32") ? 4 : 8;
name = name.Substring(0, name.Length - 2);
}
else
{
throw new Exception("Unknown field type " + name);
}
switch (name)
{
case "i": return(new IntField(structDef, size, false, false));
case "u": return(new IntField(structDef, size, true, false));
case "x": return(new IntField(structDef, size, true, true));
case "enum": return(new EnumField(structDef, size));
case "set": return(new SetField(structDef, size));
case "bits": return(new BitsField(structDef, size));
}
throw new Exception("Unknown field type " + name);
}
/// <summary>
/// Allows the algorithm to modify the attributes on a factor. For example context-specific
/// message attributes on a method invoke expression
/// </summary>
/// <param name="factorExpression">The expression</param>
/// <param name="factorAttributes">Attribute registry</param>
/// <returns></returns>
public virtual void ModifyFactorAttributes(IExpression factorExpression, AttributeRegistry <object, ICompilerAttribute> factorAttributes)
{
// By default, remove any message path attributes
factorAttributes.Remove <MessagePathAttribute>(factorExpression);
return;
}
protected override async Task DoTaskWork(string osmFile, AttributeRegistry attributeRegistry)
{
ExecuteSqlCmd("ALTER TABLE Node ADD CONSTRAINT PK_Node PRIMARY KEY CLUSTERED (Id) " +
"WITH( STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY];");
ExecuteSqlCmd("CREATE CLUSTERED INDEX [idxNode] ON [dbo].[NodeTag] ([NodeId] ASC,[Typ] ASC) WITH (STATISTICS_NORECOMPUTE = OFF, SORT_IN_TEMPDB = OFF, IGNORE_DUP_KEY = OFF, DROP_EXISTING = OFF, ONLINE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY]");
}
private void RunTasksParallel(IEnumerable <TaskBase> tasks, SqlConnectionStringBuilder connectionStringBuilder, string fileName, AttributeRegistry attributeRegistry)
{
var parallelFinishTasks = tasks
.Select(x => Task.Run(() => x.RunTask(connectionStringBuilder, fileName, attributeRegistry)))
.ToArray();
Task.WaitAll(parallelFinishTasks);
}
private void RunTasksSynchron(IEnumerable <TaskBase> tasks, SqlConnectionStringBuilder connectionStringBuilder, string fileName, AttributeRegistry attributeRegistry)
{
foreach (var task in tasks)
{
task.RunTask(connectionStringBuilder, fileName, attributeRegistry);
}
}
