static void Main(string[] args)
{
//Create TableColumnsMetadata if doesn't exist
CreateTable("TableColumnsMetadata");
QueryResult resulTableStorageXML = new QueryResult();
tableHelper = new TableHelper(account.Credentials.AccountName, GetValueFromConnectionString("AccountKey"));
DataPagerStorage dataPager = new DataPagerStorage(null, null, null);
do
{
resulTableStorageXML = tableHelper.QueryEntities("EntityMetadata", string.Empty, dataPager);
List <EntityMetadata> entityMetadata = EntityMetadata.LoadEntitiesFromTableStorageXML(resulTableStorageXML.Result);
if (entityMetadata.Count.Equals(0))
{
WriteInFile(filename, "No datasets in EntityMetadata");
return;
}
foreach (EntityMetadata entity in entityMetadata)
{
//for each dataset in entitymetadata generate rdf
GenerateRdf(entity);
}
if (!string.IsNullOrEmpty(resulTableStorageXML.FilterNextPartitionKey))
{
dataPager = new DataPagerStorage(resulTableStorageXML.FilterNextPartitionKey, resulTableStorageXML.FilterNextRowKey, string.Empty);
}
} while (!string.IsNullOrEmpty(resulTableStorageXML.FilterNextPartitionKey));
tw.Close();
}
public static bool GenerateTableColumnMetadata(EntityMetadata entity)
{
DataPagerStorage dataPager = new DataPagerStorage(null, null, null);
Filter filter = new Filter();
filter.Conditions = new List <Condition>();
Condition condition = new Condition();
condition.Value = entity.Entityset.ToLower();
condition.Attibute = "entityset";
condition.OperatorFilter = "eq";
filter.Conditions.Add(condition);
string strFilter = filter.GetRESTFilter();
//if exists in TableColumnsMetadata
List <Entity> tableColumnsMetadata = TableMetadata.LoadGenericEntitiesFromTableStorageXML(tableHelper.QueryEntities("TableColumnsMetadata", strFilter, dataPager).Result);
if (tableColumnsMetadata.Count.Equals(0))
{
TableColumnsMetadataItem columnsMeta;
Entity columnMetaLine;
string defaultDescription = ConfigurationManager.AppSettings["DefaultDescription"].ToString();
string defaultDescriptionToAdd = string.Empty;
try
{
foreach (DictionaryEntry column in entity.Columns)
{
defaultDescriptionToAdd = string.Format(defaultDescription, column.Key.ToString(), entity.Entityset);
columnsMeta = new TableColumnsMetadataItem(column.Key.ToString(), string.Empty, defaultDescriptionToAdd, "ogdi=ogdiUrl");
columnsMeta.EntitySet = entity.Entityset.ToLower();
columnMetaLine = GetSchemaEntity(columnsMeta);
if (!StoreEntity("TableColumnsMetadata", columnMetaLine, entity.Entityset, column.Key.ToString()))
{
WriteInFile(filename, "Error generating columns metadata for dataset " + entity.Entityset + " in column " + column.Key.ToString());
return(false);
}
}
return(true);
}
catch (Exception e)
{
WriteInFile(filename, "Error generating columns metadata for dataset " + entity.Entityset + " " + e.Message);
return(false);
}
}
else
{
//already has columns metadata
return(true);
}
}
private static void GenerateRdf(EntityMetadata entity)
{
DataPagerStorage dataPager = new DataPagerStorage(null, null, null);
QueryResult resulTableStorageXML = new QueryResult();
Entity datasetUpdated = new Entity();
try
{
do
{
resulTableStorageXML = tableHelper.QueryEntities(entity.Entityset, string.Empty, dataPager);
List <Entity> datasetMetadata = TableMetadata.LoadGenericEntitiesFromTableStorageXML(resulTableStorageXML.Result);
if (datasetMetadata.Count.Equals(0))
{
WriteInFile(filename, "Dataset " + entity.Entityset + " is empty.");
return;
}
foreach (Entity datasetLine in datasetMetadata)
{
if (datasetLine["rdfsnippet"] == null || datasetLine["rdfsnippet"].ToString() == string.Empty)
{
datasetUpdated = GenerateRdfSnippet(datasetLine, entity);
}
if (datasetUpdated == null)
{
WriteInFile(filename, "Error generating RdfSnippet for dataset " + entity.Entityset + " in line with entityid " + datasetLine["entityid"].ToString());
}
}
if (!GenerateTableColumnMetadata(entity))
{
WriteInFile(filename, "Error generating TableColumnsMetadata for dataset " + entity.Entityset);
return;
}
if (!string.IsNullOrEmpty(resulTableStorageXML.FilterNextPartitionKey))
{
dataPager = new DataPagerStorage(resulTableStorageXML.FilterNextPartitionKey, resulTableStorageXML.FilterNextRowKey, string.Empty);
}
} while (!string.IsNullOrEmpty(resulTableStorageXML.FilterNextPartitionKey));
WriteInFile(filename, "Rdf data for dataset " + entity.Entityset + " generated successfully");
}
catch (Exception e)
{
WriteInFile(filename, "Error in dataset " + entity.Entityset + " " + e.Message);
}
}
public QueryResult QueryEntities(string tableName, string filter, DataPagerStorage dataPager)
{
QueryResult queryResult = new QueryResult();
if (!string.IsNullOrWhiteSpace(filter))
{
queryResult.Filter = "$filter=" + Uri.EscapeDataString(filter);
}
if (!string.IsNullOrWhiteSpace(dataPager.NumberItems))
{
queryResult.FilterTop = "&$top=" + Uri.EscapeDataString(dataPager.NumberItems);
}
if (!string.IsNullOrWhiteSpace(dataPager.NextPartitionKey))
{
queryResult.FilterNextPartitionKey = "&NextPartitionKey=" + Uri.EscapeDataString(dataPager.NextPartitionKey);
}
if (!string.IsNullOrWhiteSpace(dataPager.NextRowKey))
{
queryResult.FilterNextRowKey = "&NextRowKey=" + Uri.EscapeDataString(dataPager.NextRowKey);
}
return(Retry <QueryResult>(delegate()
{
HttpWebRequest request;
HttpWebResponse response;
string entityXml = null;
try
{
string resource = String.Format(tableName + "?" + queryResult.Filter + queryResult.FilterTop + queryResult.FilterNextPartitionKey + queryResult.FilterNextRowKey);
request = CreateRESTRequest("GET", resource, null, null);
request.Accept = "application/atom+xml,application/xml";
response = request.GetResponse() as HttpWebResponse;
if ((int)response.StatusCode == 200)
{
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
string result = reader.ReadToEnd();
XNamespace ns = "http://www.w3.org/2005/Atom";
XNamespace d = "http://schemas.microsoft.com/ado/2007/08/dataservices";
XElement entry = XElement.Parse(result);
entityXml = entry.ToString();
}
dataPager.AddPage(response.GetResponseHeader("x-ms-continuation-NextPartitionKey"), response.GetResponseHeader("x-ms-continuation-NextRowKey"));
}
response.Close();
queryResult.Result = entityXml;
queryResult.FilterNextPartitionKey = dataPager.NextPartitionKey;
queryResult.FilterNextRowKey = dataPager.NextRowKey;
return queryResult;
}
catch (WebException ex)
{
if (ex.Status == WebExceptionStatus.ProtocolError &&
ex.Response != null &&
(int)(ex.Response as HttpWebResponse).StatusCode == 409)
{
return null;
}
throw;
}
}));
}
