public override IEnumerable<Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); do { bool isExceptionOccured = false; var entity = new Entity(); try { for (int i = 0; i < _dataReader.FieldCount; ++i) { entity.AddProperty(_dataReader.GetName(i), _dataReader[i]); } var placemark = _kml.Descendants().Where(e => e.Name.LocalName == DataLoaderConstants.ElemNamePlacemark). Where(e => e.Element(e.Name.Namespace + _params.MatchElementName) != null). Where(e => e.Element(e.Name.Namespace + _params.MatchElementName) .Value.Contains(_dataReader[_params.MatchPropertyName].ToString())) .FirstOrDefault(); if (placemark != null) { placemark = new XElement(placemark); foreach (string name in _params.KmlElementsToStrip) { if (string.IsNullOrEmpty(name)) continue; placemark.Element(string.Concat(placemark.Name.Namespace, name)).Remove(); } placemark.Element(placemark.Name.Namespace + DataLoaderConstants.ElemNameDescription).SetValue(entity.Id); var p = placemark.ToString(SaveOptions.DisableFormatting) .Replace(DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, p); } if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(entity.ToString(), ex)); isExceptionOccured = true; } if (!isExceptionOccured) { yield return entity; count++; } } while (_dataReader.Read()); }
public override IEnumerable <Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); do { bool isExceptionOccured = false; var entity = new Entity(); try { for (int i = 0; i < _dataReader.FieldCount; ++i) { entity.AddProperty(_dataReader.GetName(i), _dataReader[i]); } var placemark = _kml.Descendants().Where(e => e.Name.LocalName == DataLoaderConstants.ElemNamePlacemark). Where(e => e.Element(e.Name.Namespace + _params.MatchElementName) != null). Where(e => e.Element(e.Name.Namespace + _params.MatchElementName) .Value.Contains(_dataReader[_params.MatchPropertyName].ToString())) .FirstOrDefault(); if (placemark != null) { placemark = new XElement(placemark); foreach (string name in _params.KmlElementsToStrip) { placemark.Element(placemark.Name.Namespace + name).Remove(); } placemark.Element(placemark.Name.Namespace + DataLoaderConstants.ElemNameDescription).SetValue(entity.Id); var p = placemark.ToString(SaveOptions.DisableFormatting) .Replace(DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, p); } if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(entity.ToString(), ex)); isExceptionOccured = true; } if (!isExceptionOccured) { yield return(entity); count++; } }while (_dataReader.Read()); }
public virtual void Load(ProgressCallback progressNotifier, OnContinueExceptionCallback exceptionNotifier) { bool needLoadData = Producer != null; if (needLoadData) { Verify(); } OnLoadStart(); if (needLoadData) { #region RDF TableColumnsMetadata foreach (TableColumnsMetadataItem item in Params.TableColumnsMetadata.PropertyToTypeColumnsMetadata.Mappings) { item.EntitySet = Params.TableMetadataEntity.EntitySet; Entity TableColumnsMetadataEntity = TableColumnsMetadata.GetSchemaEntity(item); Processor.ProcessRdfMetadataEntity(DataLoaderConstants.TableColumnsMetadataTableName, TableColumnsMetadataEntity); } #endregion int count = 0; int loadThreadsCount = int.Parse(ConfigurationManager.AppSettings["LoadThreadsCount"]); var options = new ParallelOptions { MaxDegreeOfParallelism = loadThreadsCount }; Parallel.ForEach(Producer.GetEntitiesEnumerator(exceptionNotifier, Params), options, i => { try { Processor.ProcessEntity(Params.TableMetadataEntity.EntitySet, i); } catch (Exception ex) { if (exceptionNotifier != null) { exceptionNotifier(new EntityProcessingException(i.ToString(), ex)); } } if (progressNotifier != null) { progressNotifier(Producer.EntityCount, Interlocked.Increment(ref count)); } }); } Processor.ProcessTableMetadataEntity(DataLoaderConstants.EntitySetTableMetadata, Params.TableMetadataEntity); if (needLoadData) { Processor.ProcessEntityMetadataEntity(DataLoaderConstants.EntitySetEntityMetadata, Producer.SchemaEntity); } }
public virtual void Load(ProgressCallback progressNotifier, OnContinueExceptionCallback exceptionNotifier) { bool needLoadData = Producer != null; if (needLoadData) Verify(); OnLoadStart(); if (needLoadData) { #region RDF TableColumnsMetadata foreach (TableColumnsMetadataItem item in Params.TableColumnsMetadata.PropertyToTypeColumnsMetadata.Mappings) { item.EntitySet = Params.TableMetadataEntity.EntitySet; Entity TableColumnsMetadataEntity = TableColumnsMetadata.GetSchemaEntity(item); Processor.ProcessRdfMetadataEntity(DataLoaderConstants.TableColumnsMetadataTableName, TableColumnsMetadataEntity); } #endregion int count = 0; int loadThreadsCount = int.Parse(ConfigurationManager.AppSettings["LoadThreadsCount"]); var options = new ParallelOptions { MaxDegreeOfParallelism = loadThreadsCount }; Parallel.ForEach(Producer.GetEntitiesEnumerator(exceptionNotifier, Params), options, i => { try { Processor.ProcessEntity(Params.TableMetadataEntity.EntitySet, i); } catch (Exception ex) { if (exceptionNotifier != null) exceptionNotifier(new EntityProcessingException(i.ToString(), ex)); } if (progressNotifier != null) progressNotifier(Producer.EntityCount, Interlocked.Increment(ref count)); }); } Processor.ProcessTableMetadataEntity(DataLoaderConstants.EntitySetTableMetadata, Params.TableMetadataEntity); if (needLoadData) { Processor.ProcessEntityMetadataEntity(DataLoaderConstants.EntitySetEntityMetadata, Producer.SchemaEntity); } }
public override IEnumerable<Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); var properties = _params.PropertyToTypeMap.GetProperties().ToDictionary(c => c.Key.ToLower(), c => c.Value); string[] headers = _csvReader.GetFieldHeaders(); bool isInlineKmlSnippetProvided = false; #region RDF string entitySet = SchemaEntity["EntitySet"].ToString(); // rdf get the columns namespaces List<TableColumnsMetadataEntity> columnMetadata = TableDataLoader.GetRdfMetadataColumnNamespace(entitySet); List<string> namespacesRdf = new List<string>(); List<string> validNamespaces = new List<string>(); string namespaceToAdd = string.Empty; if (columnMetadata != null) { foreach (TableColumnsMetadataEntity columnMeta in columnMetadata) { if (!string.IsNullOrEmpty(columnMeta.columnnamespace)) { namespaceToAdd = columnMeta.columnnamespace.Split('=')[0] + '='; if (namespaceToAdd.Split('=')[0] != string.Empty) { if (!validNamespaces.Contains(namespaceToAdd)) { namespacesRdf.Add(columnMeta.columnnamespace); validNamespaces.Add(namespaceToAdd); } } } } } #endregion while (_csvReader.ReadNextRecord()) { var entity = new Entity(); bool isExceptionOccured = false; #region RDF XNamespace rdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; XElement rdfXml = new XElement(rdfNamespace + "RDF", new XAttribute(XNamespace.Xmlns + "rdf", rdfNamespace.ToString())); // add new namespaces to the rdf snippet if they exist if (namespacesRdf != null) { foreach (string ns in namespacesRdf) { if (!string.IsNullOrEmpty(ns)) { rdfXml.Add(new XAttribute(XNamespace.Xmlns + ns.ToString().Split('=')[0], ns.ToString().Split('=')[1])); } } } XElement rdfXmlDescriptionElement = new XElement(rdfNamespace + "Description"); rdfXml.Add(rdfXmlDescriptionElement); #endregion try { for (int i = 0; i < _csvReader.FieldCount; ++i) { if (_csvReader[i] == null) continue; if (_csvReader[i].GetType() == typeof(DBNull)) continue; var header = headers[i]; if (!isInlineKmlSnippetProvided) isInlineKmlSnippetProvided = header.Equals(DataLoaderConstants.PropNameKmlSnippet, StringComparison.InvariantCultureIgnoreCase); try { if (!properties.ContainsKey(header.ToLower())) throw new ApplicationException("The row data is mismached with column definitions in data file."); var stringType = properties[header.ToLower()]; var stringValue = _csvReader[i]; if (string.IsNullOrEmpty(stringValue)) continue; var v = GetPropertyValue(stringType, stringValue); entity.AddProperty(header, v); #region RDF if (header == Params.ProcessorParams.PartitionKeyPropertyName) { rdfXmlDescriptionElement.Add(new XAttribute(rdfNamespace + "about", stringValue)); } var datatype = GetRdfType(stringType); var cleanHeader = CleanStringLower(header); var columnNs = columnMetadata.First(column => column.column == header); XNamespace customNS = columnNs.columnnamespace.ToString().Split('=')[1]; if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader, v.ToString(), new XAttribute(rdfNamespace + "datatype", datatype))); } else rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader)); #endregion } catch (FormatException ex) { var sb = new StringBuilder(); sb.Append("Could not add property:"); sb.AppendFormat("\r\nName\t = '{0}'", header); sb.AppendFormat("\r\nType\t = '{0}'", _params.PropertyToTypeMap.GetPropertyType(header)); sb.AppendFormat("\r\nValue\t = '{0}'", _csvReader[i]); exceptionNotifier(new EntityProcessingException(sb.ToString(), ex)); } } #region RDF entity.AddProperty(DataLoaderConstants.PropNameRdfSnippet, rdfXml.ToString(SaveOptions.DisableFormatting)); #endregion if (!isInlineKmlSnippetProvided && _params.PlacemarkParams != null) { var longitude = _csvReader[_params.PlacemarkParams.LongitudeProperty]; var latitude = _csvReader[_params.PlacemarkParams.LatitudeProperty]; if (longitude.GetType() != typeof(DBNull) && latitude.GetType() != typeof(DBNull)) { decimal lon; decimal lat; FormatLonLat(out lon, out lat, longitude, latitude); if (!(lon == 0 && lat == 0)) { var placemark = new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNamePlacemark); var names = new string[_params.PlacemarkParams.NameProperties.Length]; for (int i = 0; i < _params.PlacemarkParams.NameProperties.Length; i++) names[i] = _csvReader[_params.PlacemarkParams.NameProperties[i]]; placemark.Add(new XElement( XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameName, new XCData(string.Format(_params.PlacemarkParams.NamePropertyFormatString ?? "{0}", names)))); var coords = new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameCoordinates, lon.ToString(CultureInfo.InvariantCulture) + "," + lat.ToString(CultureInfo.InvariantCulture)); placemark.Add(new XElement( XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNamePoint, coords)); placemark.Add(new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameDescription, entity.Id)); var ps = placemark.ToString(SaveOptions.DisableFormatting).Replace( DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, ps); } } } if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(entity.ToString(), ex)); isExceptionOccured = true; } if (isExceptionOccured) continue; yield return entity; count++; } }
public override IEnumerable <Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); var properties = _params.PropertyToTypeMap.GetProperties().ToDictionary(c => c.Key.ToLower(), c => c.Value); string[] headers = _csvReader.GetFieldHeaders(); bool isInlineKmlSnippetProvided = false; #region RDF string entitySet = SchemaEntity["EntitySet"].ToString(); // rdf get the columns namespaces List <TableColumnsMetadataEntity> columnMetadata = TableDataLoader.GetRdfMetadataColumnNamespace(entitySet); List <string> namespacesRdf = new List <string>(); List <string> validNamespaces = new List <string>(); string namespaceToAdd = string.Empty; if (columnMetadata != null) { foreach (TableColumnsMetadataEntity columnMeta in columnMetadata) { if (!string.IsNullOrEmpty(columnMeta.columnnamespace)) { namespaceToAdd = columnMeta.columnnamespace.Split('=')[0] + '='; if (namespaceToAdd.Split('=')[0] != string.Empty) { if (!validNamespaces.Contains(namespaceToAdd)) { namespacesRdf.Add(columnMeta.columnnamespace); validNamespaces.Add(namespaceToAdd); } } } } } #endregion while (_csvReader.ReadNextRecord()) { var entity = new Entity(); bool isExceptionOccured = false; #region RDF XNamespace rdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; XElement rdfXml = new XElement(rdfNamespace + "RDF", new XAttribute(XNamespace.Xmlns + "rdf", rdfNamespace.ToString())); // add new namespaces to the rdf snippet if they exist if (namespacesRdf != null) { foreach (string ns in namespacesRdf) { if (!string.IsNullOrEmpty(ns)) { rdfXml.Add(new XAttribute(XNamespace.Xmlns + ns.ToString().Split('=')[0], ns.ToString().Split('=')[1])); } } } XElement rdfXmlDescriptionElement = new XElement(rdfNamespace + "Description"); rdfXml.Add(rdfXmlDescriptionElement); #endregion try { for (int i = 0; i < _csvReader.FieldCount; ++i) { if (_csvReader[i] == null) { continue; } if (_csvReader[i].GetType() == typeof(DBNull)) { continue; } var header = headers[i]; if (!isInlineKmlSnippetProvided) { isInlineKmlSnippetProvided = header.Equals(DataLoaderConstants.PropNameKmlSnippet, StringComparison.InvariantCultureIgnoreCase); } try { if (!properties.ContainsKey(header.ToLower())) { throw new ApplicationException("The row data is mismached with column definitions in data file."); } var stringType = properties[header.ToLower()]; var stringValue = _csvReader[i]; if (string.IsNullOrEmpty(stringValue)) { continue; } var v = GetPropertyValue(stringType, stringValue); entity.AddProperty(header, v); #region RDF if (header == Params.ProcessorParams.PartitionKeyPropertyName) { rdfXmlDescriptionElement.Add(new XAttribute(rdfNamespace + "about", stringValue)); } var datatype = GetRdfType(stringType); var cleanHeader = CleanStringLower(header); var columnNs = columnMetadata.First(column => column.column == header); XNamespace customNS = columnNs.columnnamespace.ToString().Split('=')[1]; if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader, v.ToString(), new XAttribute(rdfNamespace + "datatype", datatype))); } else { rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader)); } #endregion } catch (FormatException ex) { var sb = new StringBuilder(); sb.Append("Could not add property:"); sb.AppendFormat("\r\nName\t = '{0}'", header); sb.AppendFormat("\r\nType\t = '{0}'", _params.PropertyToTypeMap.GetPropertyType(header)); sb.AppendFormat("\r\nValue\t = '{0}'", _csvReader[i]); exceptionNotifier(new EntityProcessingException(sb.ToString(), ex)); } } #region RDF entity.AddProperty(DataLoaderConstants.PropNameRdfSnippet, rdfXml.ToString(SaveOptions.DisableFormatting)); #endregion if (!isInlineKmlSnippetProvided && _params.PlacemarkParams != null) { var longitude = _csvReader[_params.PlacemarkParams.LongitudeProperty]; var latitude = _csvReader[_params.PlacemarkParams.LatitudeProperty]; if (longitude.GetType() != typeof(DBNull) && latitude.GetType() != typeof(DBNull)) { var formatInfo = CultureInfo.InvariantCulture.NumberFormat; var lon = decimal.Parse(longitude.Replace(',', '.'), formatInfo); var lat = decimal.Parse(latitude.Replace(',', '.'), formatInfo); if (!(lon == 0 && lat == 0)) { var placemark = new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNamePlacemark); var names = new string[_params.PlacemarkParams.NameProperties.Length]; for (int i = 0; i < _params.PlacemarkParams.NameProperties.Length; i++) { names[i] = _csvReader[_params.PlacemarkParams.NameProperties[i]]; } placemark.Add(new XElement( XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameName, new XCData(string.Format(_params.PlacemarkParams.NamePropertyFormatString ?? "{0}", names)))); var coords = new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameCoordinates, lon.ToString(CultureInfo.InvariantCulture) + "," + lat.ToString(CultureInfo.InvariantCulture)); placemark.Add(new XElement( XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNamePoint, coords)); placemark.Add(new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameDescription, entity.Id)); var ps = placemark.ToString(SaveOptions.DisableFormatting).Replace( DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, ps); } } } if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(entity.ToString(), ex)); isExceptionOccured = true; } if (isExceptionOccured) { continue; } yield return(entity); count++; } }
public abstract IEnumerable<Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params);
public override IEnumerable <Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); var properties = _params.PropertyToTypeMap.GetProperties().ToDictionary(c => c.Key.ToLower(), c => c.Value); bool isExceptionOccurred = false; var kmlNamespace = XNamespace.Get(DataLoaderConstants.NsKmlNew); string[] separators = new string[] { ",", " ", Environment.NewLine }; List <XElement> placemarks = _kml.Descendants(kmlNamespace + DataLoaderConstants.ElemNamePlacemark).ToList(); #region RDF string entitySet = SchemaEntity["EntitySet"].ToString(); // rdf get the columns namespaces List <TableColumnsMetadataEntity> columnMetadata = TableDataLoader.GetRdfMetadataColumnNamespace(entitySet); List <string> namespacesRdf = new List <string>(); List <string> validNamespaces = new List <string>(); string namespaceToAdd = string.Empty; if (columnMetadata != null) { foreach (TableColumnsMetadataEntity columnMeta in columnMetadata) { if (!string.IsNullOrEmpty(columnMeta.columnnamespace)) { namespaceToAdd = columnMeta.columnnamespace.Split('=')[0] + '='; if (namespaceToAdd.Split('=')[0] != string.Empty) { if (!validNamespaces.Contains(namespaceToAdd)) { namespacesRdf.Add(columnMeta.columnnamespace); validNamespaces.Add(namespaceToAdd); } } } } } #endregion for (var i = 0; i < placemarks.Count; i++) { var entity = new Entity(); isExceptionOccurred = false; #region RDF XNamespace rdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; XElement rdfXml = new XElement(rdfNamespace + "RDF", new XAttribute(XNamespace.Xmlns + "rdf", rdfNamespace.ToString())); // add new namespaces to the rdf snippet if they exist if (namespacesRdf != null) { foreach (string ns in namespacesRdf) { if (!string.IsNullOrEmpty(ns)) { rdfXml.Add(new XAttribute(XNamespace.Xmlns + ns.ToString().Split('=')[0], ns.ToString().Split('=')[1])); } } } XElement rdfXmlDescriptionElement = new XElement(rdfNamespace + "Description"); rdfXml.Add(rdfXmlDescriptionElement); #endregion try { XElement placemark = placemarks[i]; if (placemark == null) { continue; } foreach (var item in properties) { #region RDF var header = item.Key; var stringValue = item.Value; var rdfValue = String.Empty; if (header == Params.ProcessorParams.PartitionKeyPropertyName) { rdfXmlDescriptionElement.Add(new XAttribute(rdfNamespace + "about", stringValue)); } var datatype = GetRdfType(stringValue); var cleanHeader = CleanStringLower(header); var columnNs = columnMetadata.First(column => column.column == header); XNamespace customNS = columnNs.columnnamespace.ToString().Split('=')[1]; #endregion // Name & Description of Placemark if (item.Key == DataLoaderConstants.ElemNameName || item.Key == DataLoaderConstants.ElemNameDescription) { entity.AddProperty(item.Key, placemark.Element(kmlNamespace + item.Key).Value); #region RDF rdfValue = placemark.Element(kmlNamespace + header).Value; #endregion } if (item.Key == DataLoaderConstants.PropNameKmlCoords || (item.Key == DataLoaderConstants.PropNameLatitude && properties.ContainsKey(DataLoaderConstants.PropNameLongitude))) { if (placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePoint) != null) { var point = placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePoint); var positionTuple = point.Element(kmlNamespace + DataLoaderConstants.ElemNameCoordinates).Value.Split(','); entity.AddProperty(DataLoaderConstants.PropNameLatitude, ExtractLatitude(positionTuple)); entity.AddProperty("longitude", ExtractLongitude(positionTuple)); entity.AddProperty("altitude", ExtractAltitude(positionTuple)); //entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, // string.Concat("<Placemark>", point.ToString(SaveOptions.DisableFormatting), "</Placemark>")); #region RDF if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + "latitude", ExtractLatitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); rdfXmlDescriptionElement.Add(new XElement(customNS + "longitude", ExtractLongitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) { rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude", ExtractAltitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); } } else { rdfXmlDescriptionElement.Add(new XElement(customNS + "latitude")); rdfXmlDescriptionElement.Add(new XElement(customNS + "longitude")); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) { rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude")); } } #endregion } else if (placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePolygon) != null) { var polygon = placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePolygon); var outerBoundary = polygon.Element(kmlNamespace + DataLoaderConstants.ElemNameOuterBoundaryIs); var linearRing = outerBoundary.Element(kmlNamespace + DataLoaderConstants.ElemNameLinearRing); var polygonTupleList = linearRing.Element(kmlNamespace + DataLoaderConstants.ElemNameCoordinates).Value; entity.AddProperty(DataLoaderConstants.PropNameKmlCoords, polygonTupleList); string[] positionTuple = polygonTupleList.Split(separators, StringSplitOptions.RemoveEmptyEntries); entity.AddProperty(DataLoaderConstants.PropNameLatitude, ExtractLatitude(positionTuple)); entity.AddProperty(DataLoaderConstants.PropNameLongitude, ExtractLongitude(positionTuple)); entity.AddProperty("altitude", ExtractAltitude(positionTuple)); // entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, // string.Concat("<Placemark>", polygon.ToString(SaveOptions.DisableFormatting), "</Placemark>")); #region RDF if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLatitude, ExtractLatitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLongitude, ExtractLongitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) { rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude", ExtractAltitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); } } else { rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLatitude)); rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLongitude)); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) { rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude")); } } #endregion } } if (item.Key.StartsWith("sd0")) { var schemaData = placemark.Descendants(kmlNamespace + DataLoaderConstants.ElemNameSimpleData); var property = schemaData.FirstOrDefault(e => e.HasAttributes && e.Attributes("name").First().Value == item.Key.Remove(0, 3)); string value = (property != null) ? property.Value : string.Empty; entity.AddProperty(item.Key, GetPropertyValue(item.Value, value)); #region RDF rdfValue = GetPropertyValue(item.Value, value).ToString(); #endregion } #region RDF if (item.Key == DataLoaderConstants.ElemNameName || item.Key == DataLoaderConstants.ElemNameDescription || item.Key.StartsWith("sd0")) { if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader, rdfValue.ToString(), new XAttribute(rdfNamespace + "datatype", datatype))); } else { rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader)); } } #endregion } #region RDF entity.AddProperty(DataLoaderConstants.PropNameRdfSnippet, rdfXml.ToString(SaveOptions.DisableFormatting)); #endregion var ps = placemark.ToString(SaveOptions.DisableFormatting).Replace( DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, ps); if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(ex.Message, ex)); isExceptionOccurred = true; } if (isExceptionOccurred) { continue; } yield return(entity); count++; } }
public override IEnumerable <Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); var properties = _params.PropertyToTypeMap.GetProperties().ToDictionary(c => c.Key.ToLower(), c => c.Value); string[] headers = _csvReader.GetFieldHeaders(); while (_csvReader.ReadNextRecord()) { var entity = new Entity(); bool isExceptionOccured = false; try { for (int i = 0; i < _csvReader.FieldCount; ++i) { if (_csvReader[i].GetType() == typeof(DBNull)) { continue; } var n = headers[i]; try { if (!properties.ContainsKey(n.ToLower())) { throw new ApplicationException("The row data is mismached with column definitions in data file."); } var stringType = properties[n.ToLower()]; var stringValue = _csvReader[i]; if (string.IsNullOrEmpty(stringValue)) { continue; } var v = GetPropertyValue(stringType, stringValue); entity.AddProperty(n, v); } catch (FormatException ex) { var sb = new StringBuilder(); sb.Append("Could not add property:"); sb.AppendFormat("\r\nName\t = '{0}'", n); sb.AppendFormat("\r\nType\t = '{0}'", _params.PropertyToTypeMap[n]); sb.AppendFormat("\r\nValue\t = '{0}'", _csvReader[i]); exceptionNotifier(new EntityProcessingException(sb.ToString(), ex)); } } if (_params.PlacemarkParams != null) { var longitude = _csvReader[_params.PlacemarkParams.LongitudeProperty]; var latitude = _csvReader[_params.PlacemarkParams.LatitudeProperty]; if (longitude.GetType() != typeof(DBNull) && latitude.GetType() != typeof(DBNull)) { var lon = decimal.Parse(longitude.Replace('.', ',')); var lat = decimal.Parse(latitude.Replace('.', ',')); if (!(lon == 0 && lat == 0)) { var placemark = new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNamePlacemark); var names = new string[_params.PlacemarkParams.NameProperties.Length]; for (int i = 0; i < _params.PlacemarkParams.NameProperties.Length; i++) { names[i] = _csvReader[_params.PlacemarkParams.NameProperties[i]]; } placemark.Add(new XElement( XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameName, new XCData(string.Format(_params.PlacemarkParams.NamePropertyFormatString, names)))); var coords = new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameCoordinates, lon.ToString(CultureInfo.InvariantCulture) + "," + lat.ToString(CultureInfo.InvariantCulture)); placemark.Add(new XElement( XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNamePoint, coords)); placemark.Add(new XElement(XNamespace.Get(DataLoaderConstants.NsKmlNew) + DataLoaderConstants.ElemNameDescription, entity.Id)); var ps = placemark.ToString(SaveOptions.DisableFormatting).Replace( DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, ps); } } } if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(entity.ToString(), ex)); isExceptionOccured = true; } if (!isExceptionOccured) { yield return(entity); count++; } } }
public override IEnumerable<Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params) { int count = 0; string initialTimePrefix = GetSecondsFrom2000Prefix(); var properties = _params.PropertyToTypeMap.GetProperties().ToDictionary(c => c.Key.ToLower(), c => c.Value); bool isExceptionOccurred = false; var kmlNamespace = XNamespace.Get(DataLoaderConstants.NsKmlNew); string[] separators = new string[] { ",", " ", Environment.NewLine }; List<XElement> placemarks = _kml.Descendants(kmlNamespace + DataLoaderConstants.ElemNamePlacemark).ToList(); #region RDF string entitySet = SchemaEntity["EntitySet"].ToString(); // rdf get the columns namespaces List<TableColumnsMetadataEntity> columnMetadata = TableDataLoader.GetRdfMetadataColumnNamespace(entitySet); List<string> namespacesRdf = new List<string>(); List<string> validNamespaces = new List<string>(); string namespaceToAdd = string.Empty; if (columnMetadata != null) { foreach (TableColumnsMetadataEntity columnMeta in columnMetadata) { if (!string.IsNullOrEmpty(columnMeta.columnnamespace)) { namespaceToAdd = columnMeta.columnnamespace.Split('=')[0] + '='; if (namespaceToAdd.Split('=')[0] != string.Empty) { if (!validNamespaces.Contains(namespaceToAdd)) { namespacesRdf.Add(columnMeta.columnnamespace); validNamespaces.Add(namespaceToAdd); } } } } } #endregion for (var i = 0; i < placemarks.Count; i++) { var entity = new Entity(); isExceptionOccurred = false; #region RDF XNamespace rdfNamespace = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; XElement rdfXml = new XElement(rdfNamespace + "RDF", new XAttribute(XNamespace.Xmlns + "rdf", rdfNamespace.ToString())); // add new namespaces to the rdf snippet if they exist if (namespacesRdf != null) { foreach (string ns in namespacesRdf) { if (!string.IsNullOrEmpty(ns)) { rdfXml.Add(new XAttribute(XNamespace.Xmlns + ns.ToString().Split('=')[0], ns.ToString().Split('=')[1])); } } } XElement rdfXmlDescriptionElement = new XElement(rdfNamespace + "Description"); rdfXml.Add(rdfXmlDescriptionElement); #endregion try { XElement placemark = placemarks[i]; if (placemark == null) continue; foreach (var item in properties) { #region RDF var header = item.Key; var stringValue = item.Value; var rdfValue = String.Empty; if (header == Params.ProcessorParams.PartitionKeyPropertyName) { rdfXmlDescriptionElement.Add(new XAttribute(rdfNamespace + "about", stringValue)); } var datatype = GetRdfType(stringValue); var cleanHeader = CleanStringLower(header); var columnNs = columnMetadata.First(column => column.column == header); XNamespace customNS = columnNs.columnnamespace.ToString().Split('=')[1]; #endregion // Name & Description of Placemark if (item.Key == DataLoaderConstants.ElemNameName || item.Key == DataLoaderConstants.ElemNameDescription) { entity.AddProperty(item.Key, placemark.Element(kmlNamespace + item.Key).Value); #region RDF rdfValue = placemark.Element(kmlNamespace + header).Value; #endregion } if (item.Key == DataLoaderConstants.PropNameKmlCoords || (item.Key == DataLoaderConstants.PropNameLatitude && properties.ContainsKey(DataLoaderConstants.PropNameLongitude))) { if (placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePoint) != null) { var point = placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePoint); var positionTuple = point.Element(kmlNamespace + DataLoaderConstants.ElemNameCoordinates).Value.Split(','); entity.AddProperty(DataLoaderConstants.PropNameLatitude, ExtractLatitude(positionTuple)); entity.AddProperty("longitude", ExtractLongitude(positionTuple)); entity.AddProperty("altitude", ExtractAltitude(positionTuple)); //entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, // string.Concat("<Placemark>", point.ToString(SaveOptions.DisableFormatting), "</Placemark>")); #region RDF if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + "latitude", ExtractLatitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); rdfXmlDescriptionElement.Add(new XElement(customNS + "longitude", ExtractLongitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude", ExtractAltitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); } else { rdfXmlDescriptionElement.Add(new XElement(customNS + "latitude")); rdfXmlDescriptionElement.Add(new XElement(customNS + "longitude")); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude")); } #endregion } else if (placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePolygon) != null) { var polygon = placemark.Element(kmlNamespace + DataLoaderConstants.ElemNamePolygon); var outerBoundary = polygon.Element(kmlNamespace + DataLoaderConstants.ElemNameOuterBoundaryIs); var linearRing = outerBoundary.Element(kmlNamespace + DataLoaderConstants.ElemNameLinearRing); var polygonTupleList = linearRing.Element(kmlNamespace + DataLoaderConstants.ElemNameCoordinates).Value; entity.AddProperty(DataLoaderConstants.PropNameKmlCoords, polygonTupleList); string[] positionTuple = polygonTupleList.Split(separators, StringSplitOptions.RemoveEmptyEntries); entity.AddProperty(DataLoaderConstants.PropNameLatitude, ExtractLatitude(positionTuple)); entity.AddProperty(DataLoaderConstants.PropNameLongitude, ExtractLongitude(positionTuple)); entity.AddProperty("altitude", ExtractAltitude(positionTuple)); // entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, // string.Concat("<Placemark>", polygon.ToString(SaveOptions.DisableFormatting), "</Placemark>")); #region RDF if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLatitude, ExtractLatitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLongitude, ExtractLongitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude", ExtractAltitude(positionTuple), new XAttribute(rdfNamespace + "datatype", datatype))); } else { rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLatitude)); rdfXmlDescriptionElement.Add(new XElement(customNS + DataLoaderConstants.PropNameLongitude)); if (!string.IsNullOrEmpty(ExtractAltitude(positionTuple))) rdfXmlDescriptionElement.Add(new XElement(customNS + "altitude")); } #endregion } } if (item.Key.StartsWith("sd0")) { var schemaData = placemark.Descendants(kmlNamespace + DataLoaderConstants.ElemNameSimpleData); var property = schemaData.FirstOrDefault(e => e.HasAttributes && e.Attributes("name").First().Value == item.Key.Remove(0, 3)); string value = (property != null) ? property.Value : string.Empty; entity.AddProperty(item.Key, GetPropertyValue(item.Value, value)); #region RDF rdfValue = GetPropertyValue(item.Value, value).ToString(); #endregion } #region RDF if (item.Key == DataLoaderConstants.ElemNameName || item.Key == DataLoaderConstants.ElemNameDescription || item.Key.StartsWith("sd0")) { if (stringValue != string.Empty) { rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader, rdfValue.ToString(), new XAttribute(rdfNamespace + "datatype", datatype))); } else rdfXmlDescriptionElement.Add(new XElement(customNS + cleanHeader)); } #endregion } #region RDF entity.AddProperty(DataLoaderConstants.PropNameRdfSnippet, rdfXml.ToString(SaveOptions.DisableFormatting)); #endregion var ps = placemark.ToString(SaveOptions.DisableFormatting).Replace( DataLoaderConstants.NsKmlOld, DataLoaderConstants.NsKmlNew); entity.AddProperty(DataLoaderConstants.PropNameKmlSnippet, ps); if (_sourceOrder) { entity.SetNumber(count, initialTimePrefix); } } catch (Exception ex) { exceptionNotifier(new EntityProcessingException(ex.Message, ex)); isExceptionOccurred = true; } if (isExceptionOccurred) continue; yield return entity; count++; } }
public abstract IEnumerable <Entity> GetEntitiesEnumerator(OnContinueExceptionCallback exceptionNotifier, DataLoaderParams Params);