public static List <string> GetTableColumnsMetadata(string entitySet, string ogdiAlias)
{
var accountName =
AppSettings.EnabledStorageAccounts[ogdiAlias].storageaccountname;
var accountKey =
AppSettings.EnabledStorageAccounts[ogdiAlias].storageaccountkey;
var connString = "DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}";
var cloudStorageAccount = CloudStorageAccount.Parse(string.Format(connString, accountName, accountKey));
var context = new TableColumnsMetadataDataServiceContext(cloudStorageAccount.TableEndpoint.ToString(), cloudStorageAccount.Credentials)
{
RetryPolicy =
RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
};
List <string> namespaces = new List <string>();
var query = from entity in context.TableColumnsMetadataTable
where entity.PartitionKey == entitySet
select entity;
string namespaceAdded;
foreach (TableColumnsMetadataEntity c in query)
{
namespaceAdded = string.Empty;
namespaceAdded = namespaces.Find(item => item.Split('=')[0] == c.columnnamespace.Split('=')[0]);
if (namespaceAdded == string.Empty || namespaceAdded == null)
{
namespaces.Add(c.columnnamespace);
}
}
return(namespaces);
}
private static bool StoreEntity(string entitySetName, Entity entity, string parKey, string rowKey)
{
TableContext context = new TableContext(account.TableEndpoint.ToString(), account.Credentials)
{
RetryPolicy = RetryPolicies.RetryExponential(5, new TimeSpan(0, 0, 1))
};
TableEntity tableEntity = new TableEntity(entity, parKey, rowKey);
context.AddObject(entitySetName, tableEntity);
try
{
context.SaveChanges();
return(true);
}
catch (StorageClientException e)
{
WriteInFile(filename, "ERROR " + entitySetName + " entity was not created " + e.Message);
return(false);
}
catch (Exception e)
{
WriteInFile(filename, "ERROR " + entitySetName + " entity was not created " + e.Message);
return(false);
}
}
public static void ParallelUpload(this CloudBlockBlob blobRef, string filename, BlobRequestOptions options)
{
if (null == options)
{
options = new BlobRequestOptions()
{
Timeout = blobRef.ServiceClient.Timeout,
RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff)
};
}
// get upload history if any
UploadInfo uploadInfo = UploadInfo.LoadByUploadFilename(filename);
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
blobRef.ParallelUpload(fs, uploadInfo, options);
}
// upload completed no history needed - delete it
if (File.Exists(uploadInfo.LogFilename))
{
File.Delete(uploadInfo.LogFilename);
}
Console.WriteLine("\nUpload completed.");
}
public string AlmacenarCfdiFramework4(Stream cfdi, string Xml, string uuid)
{
var sharedAccessSignature = new StorageCredentialsSharedAccessSignature(ConfigurationManager.AppSettings["SharedAccesSignature"].Replace('|', '&'));
var blobClient = new CloudBlobClient(ConfigurationManager.AppSettings["BlobStorageEndpoint"],
sharedAccessSignature);
blobClient.RetryPolicy = RetryPolicies.RetryExponential(15, TimeSpan.FromSeconds(25));
blobClient.Timeout = TimeSpan.FromMinutes(1);
var blobContainer = blobClient
.GetContainerReference(ConfigurationManager.AppSettings["ContainerName"]);
var blob = blobContainer.GetBlobReference(uuid);
if (cfdi.Length <= MaximumBlobSizeBeforeTransmittingAsBlocks)
{
XElement xdoc = XElement.Parse(Xml);
string version = xdoc.Attribute("Version") == null ? "" : xdoc.Attribute("Version").Value;
if (version != "")
{
blob.UploadFromStream(cfdi);
blob.Metadata["versionCFDI"] = version;
blob.SetMetadata();
}
}
else
{
var blockBlob = blobContainer.GetBlockBlobReference(blob.Uri.AbsoluteUri);
blockBlob.UploadFromStream(cfdi);
}
return(blob.Uri.AbsoluteUri);
}
private List <string> GetTableColumnsMetadata()
{
TableColumnsMetadataDataServiceContext context = new TableColumnsMetadataDataServiceContext(_Account.TableEndpoint.ToString(), _Account.Credentials)
{
RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff)
};
List <string> namespaces = new List <string>();
var query = from entity in context.TableColumnsMetadataTable
where entity.PartitionKey == EntitySet
select entity;
string namespaceAdded;
foreach (var c in query)
{
namespaceAdded = string.Empty;
namespaceAdded = namespaces.Find(item => item.Split('=')[0] == c.columnnamespace.Split('=')[0]);
if (namespaceAdded == string.Empty || namespaceAdded == null)
{
namespaces.Add(c.columnnamespace);
}
}
return(namespaces);
}
private static void CheckAndUpdateMetadataLastUpdateDate(string metadataSet, string entitySet, string entityKind,
DateTime lastUpdateDate, DataLoaderParams Params)
{
var context = new TableContext(s_account.TableEndpoint.ToString(), s_account.Credentials, Params)
{
RetryPolicy =
RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
};
string query = string.Format(s_metadataEntityQueryTemplate, metadataSet, entitySet, entityKind);
List <TableEntity> results = context.Execute <TableEntity>(new Uri(query, UriKind.Relative)).ToList();
if (results.Count == 1)
{
DateTime oldLastUpdateDate = new TableMetadataEntity(results[0]).LastUpdateDate;
if (oldLastUpdateDate > lastUpdateDate)
{
throw new MetadataOutdatedException(oldLastUpdateDate, lastUpdateDate);
}
results[0].UpdateProperty(DataLoaderConstants.PropNameLastUpdateDate, lastUpdateDate);
context.UpdateObject(results[0]);
context.SaveChanges();
}
else if (results.Count > 1)
{
throw new DuplicateEntityException(query);
}
}
private static void CheckMetadataChanges(string metadataSet, string entitySet, string entityKind, Entity entity,
DataLoaderParams Params, MetadataKind metadataKind)
{
var context = new TableContext(s_account.TableEndpoint.ToString(), s_account.Credentials, Params)
{
RetryPolicy =
RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
};
string query = string.Format(s_metadataEntityQueryTemplate, metadataSet, entitySet, entityKind);
List <TableEntity> results = context.Execute <TableEntity>(new Uri(query, UriKind.Relative)).ToList();
if (results.Count == 1)
{
var exceprionColumns = new[]
{
DataLoaderConstants.PropNameEntityId,
DataLoaderConstants.PropNameLastUpdateDate
};
string differences = results[0].FindDifferences(entity, exceprionColumns);
if (differences != null)
{
throw new MetadataChangedException(entitySet, differences);
}
}
else if (results.Count > 1)
{
throw new DuplicateEntityException(query);
}
else if (results.Count == 0)
{
throw new MetadataNotFoundException(entitySet, metadataKind);
}
}
private TableServiceContext CreateContext()
{
var context = new TableServiceContext(this.account.TableEndpoint.ToString(), this.account.Credentials)
{
ResolveType = t => typeof(T),
RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff)
};
return(context);
}
public void DownloadFolderFiles(
string accountName,
string accountKey,
string containerName,
string folder,
string targetPath)
{
var storageAccount = new CloudStorageAccount(
new StorageCredentialsAccountAndKey(accountName, accountKey), false);
var blobClient = storageAccount.CreateCloudBlobClient();
blobClient.RetryPolicy = RetryPolicies.RetryExponential(3, TimeSpan.FromSeconds(10));
var container = blobClient.GetContainerReference(containerName);
container.CreateIfNotExist();
CloudBlobDirectory stagingFiles = container.GetDirectoryReference(folder);
// get all of the files in that container
//foreach (IListBlobItem blob in stagingFiles.ListBlobs())
//{
// // get the url
// var fileName = Path.GetFileName(blob.Uri.ToString());
// var localPath = Path.Combine(targetPath, fileName);
// var blockBlob = container.GetBlockBlobReference(blob.Uri.ToString());
// //blockBlob.DownloadToFile(localPath, new BlobRequestOptions() { Timeout = TimeSpan.FromSeconds(180) });
// blockBlob.ParallelDownloadToFile(localPath, 4194304);
// OnFileDownloaded(new StachFileDownloadEventArgs() { FileName = fileName });
// blockBlob = null;
//}
// limit to 2 concurrent files at a time otherwise we often end up with
// System.OutOfMemory exceptions being thrown.
ParallelOptions options = new ParallelOptions()
{
MaxDegreeOfParallelism = 2
};
Parallel.ForEach(stagingFiles.ListBlobs(), options, blob =>
{
// get the url
var fileName = Path.GetFileName(blob.Uri.ToString());
var localPath = Path.Combine(targetPath, fileName);
var blockBlob = container.GetBlockBlobReference(blob.Uri.ToString());
//blockBlob.DownloadToFile(localPath, new BlobRequestOptions() { Timeout = TimeSpan.FromSeconds(180) });
blockBlob.ParallelDownloadToFile(localPath, 4194304);
OnFileDownloaded(new StachFileDownloadEventArgs()
{
FileName = fileName
});
blockBlob = null;
});
}
/// <summary>
/// Create Context
/// </summary>
/// <returns>Table Service Context</returns>
private TableServiceContext CreateContext()
{
var context = new TableServiceContext(this.account.TableEndpoint.ToString(), this.account.Credentials)
{
ResolveType = t => typeof(TEntity),
RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff),
IgnoreResourceNotFoundException = true,
};
return(context);
}
private static TableServiceContext GetTableContext()
{
TableServiceContext tableContext;
tableContext = _tableClient.GetDataServiceContext();
tableContext.RetryPolicy = RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultMaxBackoff);
tableContext.IgnoreMissingProperties = true;
tableContext.SaveChangesDefaultOptions = SaveChangesOptions.ReplaceOnUpdate;
tableContext.IgnoreResourceNotFoundException = true;
return(tableContext);
}
private void SetupContext()
{
/*
* this retry policy will introduce a greater delay if there are retries
* than the original setting of 3 retries in 3 seconds but it will then
* show up a problem with the system without the system failing completely.
*/
this.RetryPolicy = RetryPolicies.RetryExponential(
RetryPolicies.DefaultClientRetryCount, RetryPolicies.DefaultClientBackoff);
// don't throw a DataServiceRequestException when a row doesn't exist.
this.IgnoreResourceNotFoundException = true;
}
public void UploadBlock(Stream input, long offset)
{
input.Seek(0, SeekOrigin.Begin);
string blockId = this.EncodeOffset(offset);
string contentMD5 = this.CalculateMD5(input);
var options = new BlobRequestOptions
{
RetryPolicy = RetryPolicies.RetryExponential(
RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
};
this.blob.PutBlock(blockId, input, contentMD5, options);
}
private static void DeleteRdfMetadata(string metadataSet, string entitySet,
DataLoaderParams parameters)
{
var context = new TableContext(s_account.TableEndpoint.ToString(), s_account.Credentials, parameters)
{
RetryPolicy =
RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
};
string query = string.Format(s_metadataRdfQueryTemplate, metadataSet, entitySet);
List <TableEntity> results = context.Execute <TableEntity>(new Uri(query, UriKind.Relative)).ToList();
if (results.Count > 0)
{
foreach (TableEntity i in results)
{
context.DeleteObject(i);
context.SaveChanges();
}
}
}
public void CopyFromBlob(string destinationSasUri, string srcBlobSasUri)
{
CloudBlockBlob blob = new CloudBlockBlob(srcBlobSasUri);
string fileName = (blob.Name.Contains("/"))
? blob.Name.Substring(blob.Name.LastIndexOf("/"))
: blob.Name;
CloudBlobContainer cbc = new CloudBlobContainer(destinationSasUri);
//UriBuilder ub = new UriBuilder(destUri);
//ub.Path += "/" + fileName;
//CloudBlockBlob destBlob = new CloudBlockBlob(ub.Uri);
CloudBlockBlob destBlob = cbc.GetBlockBlobReference(fileName);
BlobRequestOptions bro = new BlobRequestOptions();
bro.RetryPolicy = RetryPolicies.RetryExponential(5, TimeSpan.FromMilliseconds(150));
destBlob.BeginCopyFromBlob(blob, bro, (result) => { }, null);
// destBlob.UploadFromStream(System.IO.File.OpenRead(@"D:\Install.txt"));
System.Diagnostics.Debug.WriteLine(destBlob.Name);
}
private static void DeleteMetadata(string metadataSet, string entitySet, string entityKind,
DataLoaderParams parameters)
{
var context = new TableContext(s_account.TableEndpoint.ToString(), s_account.Credentials, parameters)
{
RetryPolicy =
RetryPolicies.RetryExponential(RetryPolicies.DefaultClientRetryCount,
RetryPolicies.DefaultClientBackoff)
};
string query = string.Format(s_metadataEntityQueryTemplate, metadataSet, entitySet, entityKind);
List <TableEntity> results = context.Execute <TableEntity>(new Uri(query, UriKind.Relative)).ToList();
if (results.Count == 1)
{
context.DeleteObject(results.First());
context.SaveChanges();
}
else if (results.Count > 1)
{
throw new DuplicateEntityException(query);
}
}
/// <summary>
/// This is where we get the role instance configured and ready to begin processing
/// STAHC jobs
/// </summary>
/// <returns>True if succesfully configured. False otherwise</returns>
public override bool OnStart()
{
ServicePointManager.DefaultConnectionLimit = 64;
DiagnosticMonitorConfiguration config = DiagnosticMonitor.GetDefaultInitialConfiguration();
config.PerformanceCounters.DataSources.Add(
new PerformanceCounterConfiguration()
{
CounterSpecifier = @"\Processor(_Total)\% Processor Time",
SampleRate = TimeSpan.FromSeconds(30)
});
config.PerformanceCounters.DataSources.Add(
new PerformanceCounterConfiguration()
{
CounterSpecifier = @"\Network Interface(*)\Bytes Total/sec",
SampleRate = TimeSpan.FromSeconds(30)
});
config.DiagnosticInfrastructureLogs.ScheduledTransferPeriod = System.TimeSpan.FromMinutes(5);
config.DiagnosticInfrastructureLogs.ScheduledTransferLogLevelFilter = LogLevel.Error;
config.Logs.ScheduledTransferPeriod = System.TimeSpan.FromMinutes(5);
config.Logs.ScheduledTransferLogLevelFilter = LogLevel.Verbose;
config.PerformanceCounters.ScheduledTransferPeriod = System.TimeSpan.FromMinutes(1);
config.WindowsEventLog.ScheduledTransferPeriod = System.TimeSpan.FromMinutes(5);
DiagnosticMonitor.Start("DiagnosticsConnectionString", config);
// restart the role upon all configuration changes
RoleEnvironment.Changing += RoleEnvironmentChanging;
// read storage account configuration settings
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
configSetter(RoleEnvironment.GetConfigurationSettingValue(configName));
});
// get the scratch path
scratchPath = RoleEnvironment.GetLocalResource(Constants.AzureScratchName).RootPath;
// get the time to sleep between runs of the queue monitoring loop
queueSleepTime = int.Parse(
RoleEnvironment.GetConfigurationSettingValue("QueueSleepTime"),
CultureInfo.InvariantCulture);
// get the max time (seconds) that the server should take to process a queue job
maxJobLength = int.Parse(
RoleEnvironment.GetConfigurationSettingValue("MaxJobLength"),
CultureInfo.InvariantCulture);
// get the storage container to be used for processing job data
jobContainer = RoleEnvironment.GetConfigurationSettingValue("JobContainer");
// get queue data/configuration
storageAccount = CloudStorageAccount.FromConfigurationSetting("DataConnectionString");
// get the name of the queue used for this job set
var queueName = RoleEnvironment.GetConfigurationSettingValue("StachQueueName");
// get the queues
CloudQueueClient queueStorage = storageAccount.CreateCloudQueueClient();
queueStorage.RetryPolicy = RetryPolicies.RetryExponential(3, TimeSpan.FromSeconds(10));
stahcJobQueue = queueStorage.GetQueueReference(queueName);
stahcJobQueue.CreateIfNotExist();
// report on read values
Trace.WriteLine(string.Format("QueueSleepTime: '{0}'",
queueSleepTime.ToString(CultureInfo.InvariantCulture)), "Verbose");
Trace.WriteLine(string.Format("MaxJobLength: '{0}'",
maxJobLength.ToString(CultureInfo.InvariantCulture)), "Verbose");
Trace.WriteLine(string.Format("JobContainer: '{0}'", jobContainer), "Verbose");
Trace.WriteLine(string.Format("StachQueueName: '{0}'", queueName), "Verbose");
// read-in/download all source files
DownloadStagingFiles();
// loop through and execute each of the actions (if any) provided in the staging file
var stagingControlFile = Path.Combine(
scratchPath,
Constants.StagingActionsFileName);
if (File.Exists(stagingControlFile))
{
var sucessful = RunStagingActions(stagingControlFile);
if (!sucessful)
{
Trace.TraceError(
"Unable to complete staging actions. Review logs for more detail.");
return(sucessful);
}
}
return(base.OnStart());
}
/// <summary>
/// Gets the default retry policy.
/// </summary>
public static RetryPolicy GetDefaultRetryPolicy()
{
return(RetryPolicies.RetryExponential(10, TimeSpan.FromSeconds(0.5)));
}
private void StoreEntity(string entitySetName, string parKeyPropName, string rowKeyPropName, Entity entity)
{
var account =
CloudStorageAccount.Parse(ConfigurationManager.AppSettings["DataConnectionString"]);
var context = new TableContext(account.TableEndpoint.ToString(), account.Credentials, _parameters)
{
RetryPolicy = RetryPolicies.RetryExponential(5, new TimeSpan(0, 0, 1))
};
var kmlSnippet = (string)entity[DataLoaderConstants.PropNameKmlSnippet];
if (kmlSnippet != null && kmlSnippet.Length > 32 * 1024)
{
string blobName = Guid.NewGuid().ToString();
string containerName = entitySetName.ToLower();
StoreKmlSnippetAsBlob(containerName, blobName, kmlSnippet);
entity[DataLoaderConstants.PropNameKmlSnippet] = string.Format(DataLoaderConstants.KmlSnippetReference,
containerName, blobName);
}
var kmlCoords = (string)entity[DataLoaderConstants.PropNameKmlCoords];
if (kmlCoords != null && kmlCoords.Length > 32 * 1024)
{
string blobName = Guid.NewGuid().ToString();
string containerName = entitySetName.ToLower();
StoreKmlSnippetAsBlob(containerName, blobName, kmlCoords);
entity[DataLoaderConstants.PropNameKmlCoords] = string.Format(DataLoaderConstants.KmlSnippetReference,
containerName, blobName);
}
TableEntity tableEntity = null;
bool isUpdate = false;
if (_parameters != null)
{
string parKey;
object pk = entity[parKeyPropName];
if (string.IsNullOrEmpty(entity.Number))
{
parKey = (pk != null) ? ConvertToProperPartitionKey(ConvertValueToString(pk)) : entity.Id.ToString();
}
else
{
parKey = entity.Number;
}
string rowKey = null;
object rk = entity[rowKeyPropName];
if (rowKeyPropName.ToLower() == DataLoaderConstants.ValueUniqueAutoGen)
{
rowKey = Guid.NewGuid().ToString();
}
else
{
rowKey = (rk != null) ? ConvertValueToString(entity[rowKeyPropName]) : Guid.NewGuid().ToString();
}
//try to load entity from storage
if (_overwriteMode == TableOverwriteMode.Add || _overwriteMode == TableOverwriteMode.Update)
{
tableEntity = LoadEntity(context, entitySetName, rowKeyPropName, rowKey, rk, parKeyPropName, parKey, pk);
if (tableEntity != null && _overwriteMode == TableOverwriteMode.Add)
{
throw new EntityAlreadyExistsException(entitySetName, rowKeyPropName, rowKey, parKeyPropName,
parKey);
}
if (tableEntity != null)
{
tableEntity.UpdateEntity(entity);
isUpdate = true;
}
}
//if not found, create new
if (tableEntity == null)
{
tableEntity = new TableEntity(entity, parKey, rowKey);
}
}
else
{
tableEntity = new TableEntity(entity);
}
if (!isUpdate)
{
context.AddObject(entitySetName, tableEntity);
}
else
{
context.UpdateObject(tableEntity);
}
try
{
context.SaveChanges();
}
catch (StorageClientException e)
{
if (e.ErrorCode == StorageErrorCode.ResourceAlreadyExists && e.StatusCode == HttpStatusCode.Conflict)
{
throw new DuplicateEntityException(tableEntity.ToString(), e);
}
}
catch (DataServiceRequestException e)
{
if (e.InnerException != null &&
((DataServiceClientException)e.InnerException).StatusCode == (int)HttpStatusCode.Conflict)
{
throw new DuplicateEntityException(tableEntity.ToString(), e);
}
}
}
