public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; DiagnosticMonitor.Start("DiagnosticsConnectionString"); // For information on handling configuration changes // see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357. RoleEnvironment.Changing += RoleEnvironmentChanging; // read storage account configuration settings CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => configSetter(RoleEnvironment.GetConfigurationSettingValue(configName))); var storageAccount = CloudStorageAccount.FromConfigurationSetting("DataConnectionString"); // initialize blob storage CloudBlobClient blobStorage = storageAccount.CreateCloudBlobClient(); container = blobStorage.GetContainerReference("converteddata"); // initialize queue storage CloudQueueClient queueStorage = storageAccount.CreateCloudQueueClient(); queue = queueStorage.GetQueueReference("workercommands"); Trace.TraceInformation("Creating container and queue..."); bool storageInitialized = false; while (!storageInitialized) { try { // create the blob container and allow public access container.CreateIfNotExist(); var permissions = container.GetPermissions(); permissions.PublicAccess = BlobContainerPublicAccessType.Container; container.SetPermissions(permissions); // create the message queue queue.CreateIfNotExist(); storageInitialized = true; } catch (StorageClientException e) { if (e.ErrorCode == StorageErrorCode.TransportError) { Trace.TraceError("Storage services initialization failure. " + "Check your storage account configuration settings. If running locally, " + "ensure that the Development Storage service is running. Message: '{0}'", e.Message); Thread.Sleep(5000); } else { throw; } } } return base.OnStart(); }
private static void UploadBlob(CloudBlobContainer container, string blobName, string filename) { CloudBlob blob = container.GetBlobReference(blobName); using (FileStream fileStream = File.OpenRead(filename)) blob.UploadFromStream(fileStream); }
public void Sync(string localPath, CloudBlobContainer blobContainer, IEnumerable<string> directoriesToExclude) { this._localPath = localPath; this._container = blobContainer; this._directoriesToExclude = directoriesToExclude; if (Monitor.TryEnter(this._syncLock)) // avoid concurrent updates from starting on this instance { try { HashSet<string> umbracoSettings = new HashSet<string>(); HashSet<string> seen = new HashSet<string>(); HashSet<string> newCerts = new HashSet<string>(); this.SyncLocalToBlob(seen, umbracoSettings); //Time to check on blob storge for updates to sync seen = new HashSet<string>(); this.SyncBlobToLocal(seen, newCerts); this.UpdateInstanceHostsFile(); this.UpdateUmbracoSettings(umbracoSettings); this.UpdateSites(newCerts); } //end sync try {} block finally { Monitor.Exit(this._syncLock); } } else { //skip sync } }
public Storage(CloudStorageAccount account) { this.account = account; fileStorage = account.CreateCloudBlobClient(); /* Initialize file container */ container = fileStorage.GetContainerReference(filesContainer); container.CreateIfNotExist(); var permissions = container.GetPermissions(); /* Full permissions. From MSDN, Container-level public access. Anonymous clients can * read blob content and metadata and container metadata, and can list the blobs within the container. * * Other alternatives are Blob (can read content but not metadata) and Off (no * anonymous access). */ // permissions.PublicAccess = BlobContainerPublicAccessType.Container; permissions.PublicAccess = BlobContainerPublicAccessType.Off; permissions.SharedAccessPolicies.Remove("basic"); permissions.SharedAccessPolicies.Add("basic", new SharedAccessPolicy() { }); container.SetPermissions(permissions); /* Initialize table (for file metadata) */ CloudTableClient.CreateTablesFromModel( typeof(FileDataContext), account.TableEndpoint.AbsoluteUri, account.Credentials); }
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 100; //Initialize Indexer storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("CrawlerStorage")); //Initialize URL Queue urlQueueClient = storageAccount.CreateCloudQueueClient(); urlQueue = urlQueueClient.GetQueueReference("urlqueue"); if (urlQueue.CreateIfNotExist()) { //Add first URL to the queue CloudQueueMessage firstURL = new CloudQueueMessage(startURL); urlQueue.AddMessage(firstURL); } //Initialize Index Queue indexQueueClient = storageAccount.CreateCloudQueueClient(); indexQueue = indexQueueClient.GetQueueReference("indexqueue"); indexQueue.CreateIfNotExist(); //Initialize Database Blob databaseClient = storageAccount.CreateCloudBlobClient(); databaseContainer = databaseClient.GetContainerReference("wordfrequencies"); databaseContainer.CreateIfNotExist(); var permission = databaseContainer.GetPermissions(); permission.PublicAccess = BlobContainerPublicAccessType.Container; databaseContainer.SetPermissions(permission); return base.OnStart(); }
public static void MyClassInitialize(TestContext testContext) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(DevConnectionString); blobClient = storageAccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference(TestContainerName); container.CreateIfNotExist(); }
/// <summary> /// Taken from: http://blogs.msdn.com/b/windowsazurestorage/archive/2012/06/12/introducing-asynchronous-cross-account-copy-blob.aspx /// </summary> /// <param name="destContainer">The container to monitor</param> public static void MonitorCopy(CloudBlobContainer destContainer) { var pendingCopy = true; while (pendingCopy) { var destBlobList = destContainer.ListBlobs(true, BlobListingDetails.Copy); foreach (var destBlob in destBlobList.Select(dest => dest as CloudBlob)) { if (destBlob.CopyState == null) { Debug.WriteLine("BlobStorage.MonitorCopy: CopyState is null. Small sleep, then we assume it's done!"); Thread.Sleep(5000); return; } switch (destBlob.CopyState.Status) { case CopyStatus.Failed: case CopyStatus.Aborted: Debug.WriteLine("BlobStorage.MonitorCopy: Copy Failed or Aborted; restarting copy"); destBlob.StartCopyFromBlob(destBlob.CopyState.Source); break; case CopyStatus.Success: pendingCopy = false; break; } } Thread.Sleep(1000); } }
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; #if DEBUG account = CloudStorageAccount.DevelopmentStorageAccount; #else account = new CloudStorageAccount(accountAndKey, true); #endif tableContext = new TableServiceContext(account.TableEndpoint.ToString(), account.Credentials); //client = new CloudQueueClient(account.BlobEndpoint.ToString(), account.Credentials); qclient = account.CreateCloudQueueClient(); q = qclient.GetQueueReference("icd9mapplotrequests"); rows = new List<ICD9MapPlotResultEntry>(); bclient = account.CreateCloudBlobClient(); container = bclient.GetContainerReference("results"); container.CreateIfNotExist(); client = account.CreateCloudTableClient(); client.CreateTableIfNotExist("ICD9MapPlotResult"); client.CreateTableIfNotExist("DoctorDetails"); client.CreateTableIfNotExist("PatientDetails"); // For information on handling configuration changes // see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357. return base.OnStart(); }
static void Sync(CloudBlobContainer container, string rootPath) { // iterate all files from storage foreach (CloudBlob item in container.ListBlobs(new BlobRequestOptions() { UseFlatBlobListing = true })) { try { var localPath = Path.Combine(rootPath, item.Name); if (File.Exists(localPath) && File.GetLastWriteTimeUtc(localPath).ToFileTime() > item.Properties.LastModifiedUtc.ToFileTime()) continue; // local file is more recent if (!Directory.Exists(Path.GetDirectoryName(localPath))) Directory.CreateDirectory(Path.GetDirectoryName(localPath)); for (var i = 0; i < RETRY_COUNT; i++) { try { using (var fs = File.Open(localPath, FileMode.Create)) { item.DownloadToStream(fs); break; // all OK } } catch (IOException) { // retry again later System.Threading.Thread.Sleep(SLEEPMS_ONFAIL); } //catch (UnauthorizedAccessException) { // // ??? just catch this silently otherwise IIS will crash //} } } catch (Exception ex) { NLog.LogManager.GetLogger("optQuick").Error(string.Format("Unable to process path {0} => {1}", rootPath, item.Name)); NLog.LogManager.GetLogger("optQuick").Error(ex); throw; } } }
//This function is using storage client ddl of blob public byte[] DownloadBlobClient(string UserId, string BlobName) { // Retrieve storage account from connection string. Microsoft.WindowsAzure.CloudStorageAccount StorageAccount = Microsoft.WindowsAzure.CloudStorageAccount.Parse(Convert.ToString(ConfigurationManager.AppSettings["StorageConnectionString"])); // Create the blob client. Microsoft.WindowsAzure.StorageClient.CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container. Microsoft.WindowsAzure.StorageClient.CloudBlobContainer container = blobClient.GetContainerReference(UserId); // Create the container if it doesn't exist. container.CreateIfNotExist(); //Set permission to public container.SetPermissions( new Microsoft.WindowsAzure.StorageClient.BlobContainerPermissions { PublicAccess = Microsoft.WindowsAzure.StorageClient.BlobContainerPublicAccessType.Off }); // Retrieve reference to a blob named Microsoft.WindowsAzure.StorageClient.CloudBlockBlob blockBlob = container.GetBlockBlobReference(BlobName); return(blockBlob.DownloadByteArray()); }
public StatelessAzureQueueWriter(IEnvelopeStreamer streamer, CloudBlobContainer container, CloudQueue queue, string name) { _streamer = streamer; _cloudBlob = container; _queue = queue; Name = name; }
public void CreateIfNotExist(CloudBlobContainer cloudBlobContainer, bool isPublic = true) { cloudBlobContainer.CreateIfNotExist(); var permission = cloudBlobContainer.GetPermissions(); var blobContainerPermissions = new BlobContainerPermissions {PublicAccess = isPublic ? BlobContainerPublicAccessType.Container : BlobContainerPublicAccessType.Off}; cloudBlobContainer.SetPermissions(blobContainerPermissions); }
public override bool OnStart() { // Set the maximum number of concurrent connections ServicePointManager.DefaultConnectionLimit = 12; // read storage account configuration settings CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => { configSetter(RoleEnvironment.GetConfigurationSettingValue(configName)); }); var storageAccount = CloudStorageAccount.FromConfigurationSetting("DataConnectionString"); // initialize blob storage var blobStorage = storageAccount.CreateCloudBlobClient(); blobContainer = blobStorage.GetContainerReference("guestbookpics"); // initialize queue storage var queueStorage = storageAccount.CreateCloudQueueClient(); queue = queueStorage.GetQueueReference("guestbookthumbs"); Trace.TraceInformation("Creating container and queue..."); bool storageInitialized = false; while (!storageInitialized) { try { // create the blob container and allow public access this.blobContainer.CreateIfNotExist(); var permissions = this.blobContainer.GetPermissions(); permissions.PublicAccess = BlobContainerPublicAccessType.Container; this.blobContainer.SetPermissions(permissions); // create the message queue(s) this.queue.CreateIfNotExist(); storageInitialized = true; } catch (StorageClientException e) { if (e.ErrorCode == StorageErrorCode.TransportError) { Trace.TraceError( "Storage services initialization failure. " + "Check your storage account configuration settings. If running locally, " + "ensure that the Development Storage service is running. Message: '{0}'", e.Message); System.Threading.Thread.Sleep(5000); } else { throw; } } } return base.OnStart(); }
public BlobOptimisticDataStore(CloudStorageAccount account, string containerName) { var blobClient = account.CreateCloudBlobClient(); blobContainer = blobClient.GetContainerReference(containerName.ToLower()); blobContainer.CreateIfNotExist(); blobReferences = new Dictionary<string, CloudBlob>(); }
static AzureHelper() { var cloudStorageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=http;AccountName=mintchipmarket;AccountKey=6wtpT0uW75m2LsThfROjp+cMA5zWx8VllZhJ5tM7kPAZZlIbPZ0t7pIXkO0s0AnzZ4sWMnl+rc5+1KjlNWKlfA=="); var blobClient = new CloudBlobClient(cloudStorageAccount.BlobEndpoint, cloudStorageAccount.Credentials); _container = blobClient.GetContainerReference("files"); _container.CreateIfNotExist(); _container.SetPermissions(new BlobContainerPermissions() { PublicAccess = BlobContainerPublicAccessType.Off }); }
static BlobRepositary() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(OgdiConfiguration.GetValue("DataConnectionString")); // initialize blob storage CloudBlobClient blobStorage = storageAccount.CreateCloudBlobClient(); convertedData = blobStorage.GetContainerReference("converteddata"); }
/// <summary> /// Connects to the storage account and creates the default container /// </summary> private void ConnectToBlobStorage() { storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("StorageConnectionString")); blobClient = storageAccount.CreateCloudBlobClient(); blobContainer = blobClient.GetContainerReference("testcontainer"); blobContainer.CreateIfNotExist(); blobContainer.SetPermissions(new BlobContainerPermissions() { PublicAccess = BlobContainerPublicAccessType.Blob }); }
///<summary>Creates an AzureDataCache service.</summary> ///<param name="container">The CloudBlobContainer to read data from.</param> ///<param name="destination">The path to save the data. (on the local disk)</param> public AzureDataCache(CloudBlobContainer container, string destination) { if (container == null) throw new ArgumentNullException("container"); if (destination == null) throw new ArgumentNullException("destination"); this.container = container; LocalPath = destination; }
public AzureStorageClient(CloudStorageAccount account) { client = account.CreateCloudBlobClient(); client.DefaultDelimiter = "/"; container = client.GetContainerReference("documents"); container.CreateIfNotExist(); }
public AzureBlobRepository(CloudStorageAccount storageAccount, CloudBlobClient blobClient, CloudBlobContainer blobContainer) { this._storageAccount = storageAccount; this._blobClient = blobClient; this._blobContainer = blobContainer; _blobContainer.SetPermissions(GetPermissions()); }
public AzureSystemEventStore() { Streamer = new EventStreamer(new EventSerializer(MessagesProvider.GetKnownEventTypes())); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(Settings.Default.AzureConnectionString); blobClient = storageAccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference(Settings.Default.AzureContainerName); container.CreateIfNotExist(); }
/// <summary> /// Save the data table to the given azure blob. This will overwrite an existing blob. /// </summary> /// <param name="table">instance of table to save</param> /// <param name="container">conatiner</param> /// <param name="blobName">blob name</param> public static void SaveToAzureBlob(this DataTable table, CloudBlobContainer container, string blobName) { var blob = container.GetBlobReference(blobName); using (BlobStream stream = blob.OpenWrite()) using (TextWriter writer = new StreamWriter(stream)) { table.SaveToStream(writer); } }
public AzureMonthViewStore(CloudBlobContainer viewContainer) { if (viewContainer == null) { throw new ArgumentNullException("viewContainer"); } this.viewContainer = viewContainer; }
public BlobStorage() { var storageAccount = CloudStorageAccount.FromConfigurationSetting("PhotoSettings"); this.client = storageAccount.CreateCloudBlobClient(); this.container = this.client.GetContainerReference("photos"); if(this.container.CreateIfNotExist()) this.container.SetPermissions(new BlobContainerPermissions() { PublicAccess = BlobContainerPublicAccessType.Container }); }
public void deleteFromBlob(string uri, string blobname) { CloudStorageAccount storageAccount; storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("BlobConnectionString")); blobClient = storageAccount.CreateCloudBlobClient(); blobContainer = blobClient.GetContainerReference(blobname); var blob = blobContainer.GetBlobReference(uri); blob.DeleteIfExists(); }
public Azure_Stream_Factory(CloudStorageAccount account, string container_Name) { _container_Name = container_Name; _blobClient = account.CreateCloudBlobClient(); container = _blobClient.GetContainerReference(container_Name); container.CreateIfNotExist(); }
public override TaskRunner CreateBlobNotExistsTaskRunner(CloudBlobContainer blobContainer, FileInformation fileInfo) { this._statistics.BlobNotExistCount++; return new SingleActionTaskRunner(() => { _messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 0)); _fileSystem.DeleteFile(fileInfo.FullPath); _messageBus.Publish(new FileProgressedMessage(fileInfo.FullPath, 1)); }); }
private void Initialize(CloudBlobContainer container) { if (container == null) { throw new ArgumentNullException("container"); } _container = container; AzureBlobs = new Collection<AzureBlobInfo>(); }
public ContainerTreeNode(CloudBlobContainer container) : this() { Container = container; foreach (var blob in container.ListBlobs().OfType<CloudBlob>()) Nodes.Add(new BlobTreeNode(blob)); Text = container.Name; }
public void Setup() { _name = Guid.NewGuid().ToString().ToLowerInvariant(); CloudStorageAccount cloudStorageAccount = ConnectionConfig.StorageAccount; var cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient(); _blobContainer = cloudBlobClient.GetContainerReference(_name); _appendOnly = new BlobAppendOnlyStore(_blobContainer); _appendOnly.InitializeWriter(); }
public ExpenseReceiptStorage() { this.account = CloudConfiguration.GetStorageAccount(AzureConnectionStrings.DataConnection); this.containerName = AzureStorageNames.ReceiptContainerName; var client = this.account.CreateCloudBlobClient(); client.RetryPolicy = RetryPolicies.Retry(3, TimeSpan.FromSeconds(5)); this.container = client.GetContainerReference(this.containerName); }
public static Tuple <bool, Exception> IsValidAccount(string account, string accountKey) { bool accountValid = true; Exception exception = null; try { Microsoft.WindowsAzure.CloudStorageAccount storageAccount = null; Microsoft.WindowsAzure.StorageClient.CloudBlobClient blobClient = null; Microsoft.WindowsAzure.StorageClient.CloudBlobContainer container = null; storageAccount = new Microsoft.WindowsAzure.CloudStorageAccount(new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(account, accountKey), true); blobClient = storageAccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference(AzureConfigContainerName); container.CreateIfNotExist(); } catch (Exception e) { accountValid = false; exception = e; } return(new Tuple <bool, Exception>(accountValid, exception)); }
public static bool UploadConfig(string configZipPath, string AzureAccountName, string AzureAccountKey, string orgID, string studyID, string homeID, string desiredConfigFilename, NLog.Logger logger = null) { Microsoft.WindowsAzure.CloudStorageAccount storageAccount = null; Microsoft.WindowsAzure.StorageClient.CloudBlobClient blobClient = null; Microsoft.WindowsAzure.StorageClient.CloudBlobContainer container = null; Microsoft.WindowsAzure.StorageClient.CloudBlockBlob blockBlob = null; string leaseId = null; try { storageAccount = new Microsoft.WindowsAzure.CloudStorageAccount(new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(AzureAccountName, AzureAccountKey), true); blobClient = storageAccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference(AzureConfigContainerName); container.CreateIfNotExist(); blockBlob = container.GetBlockBlobReference(DesiredConfigBlobName(orgID, studyID, homeID, desiredConfigFilename)); bool blobExists = BlockBlobExists(blockBlob); if (blobExists) { leaseId = AcquireLease(blockBlob, logger); // Acquire Lease on Blob } else { blockBlob.Container.CreateIfNotExist(); } if (blobExists && leaseId == null) { if (null != logger) { logger.Error("AcquireLease on Blob: " + DesiredConfigBlobName(orgID, studyID, homeID, desiredConfigFilename) + " Failed"); } return(false); } string url = blockBlob.Uri.ToString(); if (blockBlob.ServiceClient.Credentials.NeedsTransformUri) { url = blockBlob.ServiceClient.Credentials.TransformUri(url); } var req = BlobRequest.Put(new Uri(url), AzureBlobLeaseTimeout, new Microsoft.WindowsAzure.StorageClient.BlobProperties(), Microsoft.WindowsAzure.StorageClient.BlobType.BlockBlob, leaseId, 0); using (var writer = new BinaryWriter(req.GetRequestStream())) { writer.Write(File.ReadAllBytes(configZipPath)); writer.Close(); } blockBlob.ServiceClient.Credentials.SignRequest(req); req.GetResponse().Close(); ReleaseLease(blockBlob, leaseId); // Release Lease on Blob return(true); } catch (Exception e) { if (null != logger) { logger.ErrorException("UploadConfig_Azure, configZipPath: " + configZipPath, e); } ReleaseLease(blockBlob, leaseId); return(false); } }
public static bool DownloadConfig(string downloadedZipPath, string AzureAccountName, string AzureAccountKey, string orgID, string studyID, string homeID, string configFilename, NLog.Logger logger = null) { Microsoft.WindowsAzure.CloudStorageAccount storageAccount = null; Microsoft.WindowsAzure.StorageClient.CloudBlobClient blobClient = null; Microsoft.WindowsAzure.StorageClient.CloudBlobContainer container = null; Microsoft.WindowsAzure.StorageClient.CloudBlockBlob blockBlob = null; string leaseId = null; try { storageAccount = new Microsoft.WindowsAzure.CloudStorageAccount(new Microsoft.WindowsAzure.StorageCredentialsAccountAndKey(AzureAccountName, AzureAccountKey), true); blobClient = storageAccount.CreateCloudBlobClient(); container = blobClient.GetContainerReference(AzureConfigContainerName); if (configFilename == PackagerHelper.ConfigPackagerHelper.actualConfigFileName) { blockBlob = container.GetBlockBlobReference(ActualConfigBlobName(orgID, studyID, homeID, configFilename)); } else if (configFilename == PackagerHelper.ConfigPackagerHelper.desiredConfigFileName) { blockBlob = container.GetBlockBlobReference(DesiredConfigBlobName(orgID, studyID, homeID, configFilename)); } bool blobExists = BlockBlobExists(blockBlob); if (blobExists) { leaseId = AcquireLease(blockBlob, logger); // Acquire Lease on Blob } else { return(false); } if (blobExists && leaseId == null) { if (null != logger) { logger.Error("AcquireLease on Blob: " + ActualConfigBlobName(orgID, studyID, homeID, configFilename) + " Failed"); } return(false); } string url = blockBlob.Uri.ToString(); if (blockBlob.ServiceClient.Credentials.NeedsTransformUri) { url = blockBlob.ServiceClient.Credentials.TransformUri(url); } var req = BlobRequest.Get(new Uri(url), AzureBlobLeaseTimeout, null, leaseId); blockBlob.ServiceClient.Credentials.SignRequest(req); using (var reader = new BinaryReader(req.GetResponse().GetResponseStream())) { FileStream zipFile = new FileStream(downloadedZipPath, FileMode.OpenOrCreate); reader.BaseStream.CopyTo(zipFile); zipFile.Close(); } req.GetResponse().GetResponseStream().Close(); ReleaseLease(blockBlob, leaseId); // Release Lease on Blob return(true); } catch (Exception e) { if (null != logger) { logger.ErrorException("DownloadConfig_Azure, downloadZipPath: " + downloadedZipPath, e); } ReleaseLease(blockBlob, leaseId); return(false); } }