public string UploadFolder(string containerName, string localStoragePath, string prefixAzureDirectoryName) { try { LocalResource Storage = RoleEnvironment.GetLocalResource("ZYLocalStorage"); string[] filePaths = Directory.GetFiles(localStoragePath); foreach (var item in filePaths) { if (!Path.GetExtension(item).Equals(".zip")) { UploadFile(item, Path.GetFileName(item), prefixAzureDirectoryName); } } var folder = new DirectoryInfo(localStoragePath); var subFolders = folder.GetDirectories(); foreach (var directoryInfo in subFolders) { UploadFolder("zycontainer", Storage.RootPath + directoryInfo.Name, prefixAzureDirectoryName + "/" + directoryInfo.Name); } return("Upload Directory seccussfully."); } catch (Exception e) { return(e.Message); } }
private void MountSnapshot(string snapshotUri) { string storageAccountName = txtStorageAccountName.Text; string storageAccountKey = txtStorageAccountKey.Text; try { LocalResource resource = WindowsAzureSystemHelper.GetLocalStorageResource(LOCAL_STORAGE_NAME); int cacheSize = int.Parse(txtCacheSize.Text); if (resource.MaximumSizeInMegabytes < cacheSize) { throw new Exception(String.Format("Cache size {0} MB cannot be more than maximum permitted size {1} MB", cacheSize, resource.MaximumSizeInMegabytes)); } bool wasAlreadyMounted; string driveName = WADriveHelper.MountAzureDrive(storageAccountName, storageAccountKey, new Uri(snapshotUri), LocalStoragePath + txtCacheDirName.Text + ".snapshot", int.Parse(txtCacheSize.Text), int.Parse(txtDriveSize.Text), DriveMountOptions.None, out wasAlreadyMounted); lblMsg.Text = String.Format("Mounted Snapshot {0} to Drive {1}", snapshotUri, driveName); ListAllMountedDrives(); } catch (Exception ex) { lblMsg.Text = ex.Message; } }
public static void Initialize() { //First get a reference to the local file structure. localStorage = RoleEnvironment.GetLocalResource("scratchpad"); //Check to see if we have already grabbed the component files. if (!CloudBackedStore.DirectoryExists(ComponetDir)) { var url = string.Format("http://{0}.blob.core.windows.net/components/", RoleEnvironment.GetConfigurationSettingValue("StorageName")); foreach (var app in OdpiAppRepo.Apps) { if( !string.IsNullOrEmpty(app.PackageName) ) CloudBackedStore.Grab(ComponetDir, "\\" + app.PackageName, "components", url + app.PackageName); if (!string.IsNullOrEmpty(app.ConfName)) CloudBackedStore.Grab(ComponetDir, "\\" + app.ConfName, "components", url + app.ConfName); var req = app.RequiredFiles; if (req != null) { foreach (var file in req) { CloudBackedStore.Grab(ComponetDir, "\\" + file, "components", url + file); } } } } }
public async Task <IEnumerable <LocalResource> > SearchPhoneNumbersAsync(string areaCode = "415") { var localNumbers = await LocalResource.ReadAsync( pathCountryCode : "US", areaCode : int.Parse(areaCode), client : _client); return(localNumbers.ToList()); }
//Download many blobs in one folder of container to local storage public void DownloadToLocalStorage(LocalResource mystorage, string folderNameInContainer, string fileRootPath) { string blobPrefix = folderNameInContainer + "/"; bool useFlatBlobListing = false; var blobs = Container.ListBlobs(blobPrefix, useFlatBlobListing, BlobListingDetails.None); var filesOrFoders = blobs.ToList(); //Create direcotry in local storage Directory.CreateDirectory(fileRootPath); string fileOrFolderName; //If it is file, download it. If it is directory, use this function again and add this folder name after original folder name foreach(var fileOrFolder in filesOrFoders) { if(fileOrFolder.GetType() != typeof(CloudBlobDirectory)) { fileOrFolderName = Path.GetFileName(fileOrFolder.Uri.ToString()); DownloadBlob(Path.Combine(folderNameInContainer, fileOrFolderName), Path.Combine(fileRootPath, fileOrFolderName)); } else if(fileOrFolder.GetType() == typeof(CloudBlobDirectory)) { fileOrFolderName = fileOrFolder.Uri.ToString(); fileOrFolderName = fileOrFolderName.Substring(0, fileOrFolderName.Length - 1); fileOrFolderName = fileOrFolderName.Substring(fileOrFolderName.LastIndexOf("/") + 1); //Call this function again DownloadToLocalStorage(mystorage, folderNameInContainer + "/" + fileOrFolderName, Path.Combine(fileRootPath, fileOrFolderName)); } } }
internal Task <IDiscoveryResource> PublishAsync( string category, string connection, int expirySeconds, IReadOnlyDictionary <string, string> attributes = null, CancellationToken token = default) { var attrs = new Dictionary <string, string>(); if (attributes != null) // copy so user can't change them behind our back { foreach (var kvp in attributes) { attrs[kvp.Key] = kvp.Value; } } var resource = new LocalResource(category, connection, expirySeconds, attrs); resource.Updated = OnResourceUpdated; lock (this) { localResources_.Add(resource); // new local resources get a new guid, always unique UpdateAnnounceTimer(); } return(Task.FromResult((IDiscoveryResource)resource)); }
SKBitmap GetCoverBitmap() { using (Stream stream = LocalResource.GetStream("cover.png", this)) { return(SKBitmap.Decode(stream)); }; }
public Worker(string name) { Name = name; LocalResource azureLocalResource = RoleEnvironment.GetLocalResource("LocalStorage"); WorkDirectoryName = Path.Combine(azureLocalResource.RootPath, "WD" + name); /*** Init ***/ // On instancie un objet de chaque classe héritant de BusinessProcess _employees = new List <IBusinessProcess>(); Object[] args = new object[] { WorkDirectoryName }; foreach (Type instance in typeof(BusinessProcess).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BusinessProcess)))) { _employees.Add(Activator.CreateInstance(instance, args) as IBusinessProcess); } //// TODO : find a good time int maximumTimeToSleepInSeconde; if (!int.TryParse(RoleEnvironment.GetConfigurationSettingValue("MaximumSecondesTimeSleep"), out maximumTimeToSleepInSeconde)) { maximumTimeToSleepInSeconde = 2 * 60 * 60; // 2 heures } CustomSleeper = new Sleeper(2, maximumTimeToSleepInSeconde); CountLoop = 0; }
public IncomingPhoneNumberResource Hop( string twilioAccountSid, string relaySid, ILogging logging) { var original = IncomingPhoneNumberResource.Fetch(relaySid); string relayPhoneNumber = original.PhoneNumber.ToString(); logging.Log($"Releasing phone number {relayPhoneNumber} {relaySid}"); IncomingPhoneNumberResource.Delete(relaySid); var localAvailableNumbers = LocalResource.Read("US"); var desiredNumber = localAvailableNumbers.First(); logging.Log($"Purchasing new phone number {desiredNumber.PhoneNumber}"); var newNumber = IncomingPhoneNumberResource.Create( twilioAccountSid, desiredNumber.PhoneNumber, smsMethod: original.SmsMethod, smsUrl: original.SmsUrl); logging.Log($"New phone number purchased {newNumber.PhoneNumber} {newNumber.Sid}"); return(newNumber); }
public static void Initialize() { //First get a reference to the local file structure. localStorage = RoleEnvironment.GetLocalResource("scratchpad"); //Check to see if we have already grabbed the component files. if (!CloudBackedStore.DirectoryExists(ComponetDir)) { var url = string.Format("http://{0}.blob.core.windows.net/components/", RoleEnvironment.GetConfigurationSettingValue("StorageName")); foreach (var app in OdpiAppRepo.Apps) { if (!string.IsNullOrEmpty(app.PackageName)) { CloudBackedStore.Grab(ComponetDir, "\\" + app.PackageName, "components", url + app.PackageName); } if (!string.IsNullOrEmpty(app.ConfName)) { CloudBackedStore.Grab(ComponetDir, "\\" + app.ConfName, "components", url + app.ConfName); } var req = app.RequiredFiles; if (req != null) { foreach (var file in req) { CloudBackedStore.Grab(ComponetDir, "\\" + file, "components", url + file); } } } } }
public async Task tryPutResourceIn <T>(string url, Action <byte[]> loadAction) { bool isCached; string localpath; // var result = await Task.Run(() => tryReadFileFromLocalCache(url, out isCached, out localpath)); LocalResource localResource = await tryReadFileFromLocalCache(url); var result = localResource.Bytes; isCached = localResource.InCache; localpath = localResource.LocalPath; if (!isCached) { try { var request = (HttpWebRequest)WebRequest.Create(url); request.Method = "GET"; await makeRequest(request, (stream, lpath) => { var s = new MemoryStream(); stream.CopyTo(s); result = s.ToArray(); }, err => Debug.WriteLine(err.ToString()), true); } catch { result = null; } } loadAction(result); }
public static String GetLocalStoragePath() { //returns the full path of the local storage LocalResource l = RoleEnvironment.GetLocalResource("LocalSoundStore"); return(string.Format(l.RootPath)); }
public virtual void TestGetActualPath() { Configuration conf = new Configuration(); conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true); LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>(); // give public visibility Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility .Public); Path localPath = new Path("foo.jar"); string user = "******"; SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>(); FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>(); FileSystem localFs = Org.Mockito.Mockito.Mock <FileSystem>(); // stub it to return a status that indicates a directory FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>(); Org.Mockito.Mockito.When(status.IsDirectory()).ThenReturn(true); Org.Mockito.Mockito.When(localFs.GetFileStatus(localPath)).ThenReturn(status); SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf, scmClient, fs, localFs); Path actualPath = spied.GetActualPath(); NUnit.Framework.Assert.AreEqual(actualPath.GetName(), localPath.GetName()); NUnit.Framework.Assert.AreEqual(actualPath.GetParent().GetName(), localPath.GetName ()); }
async void OnSourceChanged() { if (Source.Contains("http")) { using (Stream stream = await httpClient.GetStreamAsync(Source)) { using (MemoryStream memStream = new MemoryStream()) { await stream.CopyToAsync(memStream); memStream.Seek(0, SeekOrigin.Begin); skSvg.Load(memStream); } } } else { using (var memStream = LocalResource.GetStream(Source)) { memStream.Seek(0, SeekOrigin.Begin); skSvg.Load(memStream); } } InvalidateSurface(); }
void Application_Start(object sender, EventArgs e) { // Code that runs on application startup BundleConfig.RegisterBundles(BundleTable.Bundles); AuthConfig.RegisterOpenAuth(); if (imageStorePath == null) { ImageStorePath = WebConfigurationManager.AppSettings["ImageStorePath"]; } // initialize storage account configuration setting publisher CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) => { string connectionString = RoleEnvironment.GetConfigurationSettingValue(configName); configSetter(connectionString); }); try { // initialize the local cache for the Azure drive LocalResource cache = RoleEnvironment.GetLocalResource("LocalDriveCache"); CloudDrive.InitializeCache(cache.RootPath + "cache", cache.MaximumSizeInMegabytes); // retrieve storage account CloudStorageAccount account = CloudStorageAccount.FromConfigurationSetting("DataConnectionString"); // retrieve URI for the page blob that contains the cloud drive from configuration settings string imageStoreBlobUri = RoleEnvironment.GetConfigurationSettingValue("ImageStoreBlobUri"); // unmount any previously mounted drive. foreach (var drive in CloudDrive.GetMountedDrives()) { var mountedDrive = new CloudDrive(drive.Value, account.Credentials); mountedDrive.Unmount(); } // create the Windows Azure drive and its associated page blob CloudDrive imageStoreDrive = account.CreateCloudDrive(imageStoreBlobUri); if (CloudDrive.GetMountedDrives().Count == 0) { try { imageStoreDrive.Create(16); } catch (CloudDriveException) { // drive already exists } } // mount the drive and initialize the application with the path to the image store on the Azure drive Global.ImageStorePath = imageStoreDrive.Mount(cache.MaximumSizeInMegabytes / 2, DriveMountOptions.None); } catch (CloudDriveException driveException) { Trace.WriteLine("Error: " + driveException.Message); } }
/// <exception cref="System.IO.IOException"/> private void AddToLocalResources(FileSystem fs, string fileSrcPath, string fileDstPath , string appId, IDictionary <string, LocalResource> localResources, string resources ) { string suffix = appName + "/" + appId + "/" + fileDstPath; Path dst = new Path(fs.GetHomeDirectory(), suffix); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { ostream = FileSystem.Create(fs, dst, new FsPermission((short)0x1c8)); ostream.WriteUTF(resources); } finally { IOUtils.CloseQuietly(ostream); } } else { fs.CopyFromLocalFile(new Path(fileSrcPath), dst); } FileStatus scFileStatus = fs.GetFileStatus(dst); LocalResource scRsrc = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI (dst.ToUri()), LocalResourceType.File, LocalResourceVisibility.Application, scFileStatus .GetLen(), scFileStatus.GetModificationTime()); localResources[fileDstPath] = scRsrc; }
async Task LocalFetch() { await Task.Delay(1000); var content = LocalResource.GetContent("product-by-category.json"); var listProductByCategory = JsonSerializer.Deserialize <ProductByCategory[]>(content, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); var _categories = new List <CategoryModel> { new() { Name = "All" } }; _categories.AddRange(listProductByCategory.Select(_ => new CategoryModel { Name = _.Category })); categories = _categories.ToArray(); products = listProductByCategory .SelectMany(_ => _.Products .Select(product => new ProductModel { Title = product.Title, Summary = product.Summary, Description = product.Description, Price = $"${product.Price:N2}", UrlImage = product.UrlImage, Order = product.Order, Category = _.Category, CategoryTag = _.Category.ToUpper(), })) .OrderBy(_ => _.Order) .ToArray(); }
public void RemoveObserver(String clientID, LocalResource resource) { IDictionary <String, Observationship> resourceObservers = null; IDictionary <String, Observationship> clientObservers = null; String path = resource.Path; if (_observersByResource.ContainsKey(path)) { resourceObservers = _observersByResource[path]; } if (_observersByClient.ContainsKey(clientID)) { clientObservers = _observersByClient[clientID]; } if (resourceObservers != null && clientObservers != null) { lock (_sync) { if (resourceObservers.Remove(clientID) && clientObservers.Remove(path)) { if (log.IsInfoEnabled) { log.Info(String.Format("Terminated observing relationship by GET: {0} @ {1}", clientID, path)); } return; } } } if (log.IsWarnEnabled) { log.Warn(String.Format("Cannot find observing relationship: {0} @ {1}", clientID, path)); } }
/// <exception cref="System.Exception"/> public virtual void TestSetupDistributedCacheConflictsFiles() { Configuration conf = new Configuration(); conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem )); URI mockUri = URI.Create("mockfs://mock/"); FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem (); URI file = new URI("mockfs://mock/tmp/something.zip#something"); Path filePath = new Path(file); URI file2 = new URI("mockfs://mock/tmp/something.txt#something"); Path file2Path = new Path(file2); Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath); Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path); DistributedCache.AddCacheFile(file, conf); DistributedCache.AddCacheFile(file2, conf); conf.Set(MRJobConfig.CacheFileTimestamps, "10,11"); conf.Set(MRJobConfig.CacheFilesSizes, "10,11"); conf.Set(MRJobConfig.CacheFileVisibilities, "true,true"); IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); NUnit.Framework.Assert.AreEqual(1, localResources.Count); LocalResource lr = localResources["something"]; //First one wins NUnit.Framework.Assert.IsNotNull(lr); NUnit.Framework.Assert.AreEqual(10l, lr.GetSize()); NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp()); NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> internal static LocalResource CreateZipFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) { byte[] bytes = new byte[len]; r.NextBytes(bytes); FilePath archiveFile = new FilePath(p.ToUri().GetPath() + ".ZIP"); archiveFile.CreateNewFile(); ZipOutputStream @out = new ZipOutputStream(new FileOutputStream(archiveFile)); @out.PutNextEntry(new ZipEntry(p.GetName())); @out.Write(bytes); @out.CloseEntry(); @out.Close(); LocalResource ret = recordFactory.NewRecordInstance <LocalResource>(); ret.SetResource(ConverterUtils.GetYarnUrlFromPath(new Path(p.ToString() + ".ZIP") )); ret.SetSize(len); ret.SetType(LocalResourceType.Archive); ret.SetVisibility(vis); ret.SetTimestamp(files.GetFileStatus(new Path(p.ToString() + ".ZIP")).GetModificationTime ()); return(ret); }
/// <exception cref="System.Exception"/> public virtual void TestUniqueDestinationPath() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); Path p = new Path(basedir, "dir" + 0 + ".jar"); LocalResourceVisibility vis = LocalResourceVisibility.Private; LocalResource rsrc = CreateJar(files, p, vis); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); Future <Path> rPath = singleThreadedExec.Submit(fsd); singleThreadedExec.Shutdown(); while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(rPath.IsDone()); // Now FSDownload will not create a random directory to localize the // resource. Therefore the final localizedPath for the resource should be // destination directory (passed as an argument) + file name. NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent()); }
public static string Mount() { CloudStorageAccount storageAccount; if (RoleEnvironment.IsEmulated) { storageAccount = CloudStorageAccount.DevelopmentStorageAccount; } else { storageAccount = CloudStorageAccount.Parse(connectionString); } LocalResource localCache = RoleEnvironment.GetLocalResource("InstanceDriveCache"); CloudDrive.InitializeCache(localCache.RootPath, localCache.MaximumSizeInMegabytes); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); blobClient.GetContainerReference("drives").CreateIfNotExist(); myCloudDrive = storageAccount.CreateCloudDrive( blobClient .GetContainerReference("drives") .GetPageBlobReference("mysupercooldrive.vhd") .Uri.ToString() ); myCloudDrive.CreateIfNotExist(64); return(myCloudDrive.Mount(25, DriveMountOptions.None)); }
protected override void DoExecute(Request request) { if (request != null) { String path = request.UriPath; LocalResource resource = GetResource(path); if (resource != null) { request.Resource = resource; if (log.IsDebugEnabled) { log.Debug("Dispatching execution: " + path); } _dispatcher.Dispatch(request); } else if (request.Code == Code.PUT) { // allows creation of non-existing resources through PUT CreateByPut(path, request); request.SendResponse(); } else { if (log.IsWarnEnabled) { log.Warn("Cannot find resource: " + path); } request.Respond(Code.NotFound); request.SendResponse(); } } }
public static async Task <ResourceSet <LocalResource> > GetAvailableLocalPhoneNumberAsync(ITwilioRestClient client, CountryCode countryCode, string accountSid = null, int?areaCode = null, bool?smsEnabled = null, bool?mmsEnabled = null, bool?faxEnabled = null, bool?voiceEnabled = null, bool?beta = null, string containsPattern = null, string lata = null, string latLong = null, string nearNumber = null, int?distance = null, string locality = null, string postalCode = null, string rateCenter = null, string region = null, bool?excludeAllAddressRequired = null, bool?excludeForeignAddressRequired = null, bool?excludeLocalAddressRequired = null, long?limit = null) { var nearPhoneNumber = nearNumber != null ? new PhoneNumber(nearNumber) : null; var options = new ReadLocalOptions($"{countryCode:G}") { PathAccountSid = accountSid, AreaCode = areaCode, SmsEnabled = smsEnabled, MmsEnabled = mmsEnabled, FaxEnabled = faxEnabled, VoiceEnabled = voiceEnabled, Beta = beta, Contains = containsPattern, InLata = lata, NearLatLong = latLong, NearNumber = nearPhoneNumber, Distance = distance, InLocality = locality, InPostalCode = postalCode, InRateCenter = rateCenter, InRegion = region, ExcludeAllAddressRequired = excludeAllAddressRequired, ExcludeForeignAddressRequired = excludeForeignAddressRequired, ExcludeLocalAddressRequired = excludeLocalAddressRequired, Limit = limit, PageSize = null }; return(await LocalResource.ReadAsync(options, client)); }
public void AddObserver(Request request, LocalResource resource) { request.IsObserving = true; Observationship shipToAdd = new Observationship(request); lock (_sync) { // get clients map for the given resource path IDictionary <String, Observationship> resourceObservers = SafeGet(_observersByResource, resource.Path); // get resource map for given client address IDictionary <String, Observationship> clientObservers = SafeGet(_observersByClient, request.PeerAddress.ToString()); // save relationship for notifications triggered by resource resourceObservers[request.PeerAddress.ToString()] = shipToAdd; // save relationship for actions triggered by client clientObservers[resource.Path] = shipToAdd; } if (log.IsInfoEnabled) { log.Info(String.Format("Established observing relationship: {0} @ {1}", request.PeerAddress, resource.Path)); } // update response PrepareResponse(request); }
private static DirectoryConfiguration CreateDirectoryConfiguration(LocalResource localResource) { DirectoryConfiguration dirConfig = new DirectoryConfiguration(); dirConfig.Container = "diagnostics-customlogfiles-container"; dirConfig.DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes; dirConfig.Path = localResource.RootPath; return dirConfig; }
internal MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag, URL localPath, SerializedException ex) { this.rsrc = rsrc; this.tag = tag; this.localPath = localPath; this.ex = ex; }
public void Dispose() { if (Directory.Exists(this.CurrentDirectory)) Directory.Delete(this.CurrentDirectory, true); this.CurrentDirectory = null; this._LocalResource = null; }
public async Task <ResourceDto> GetAsync(long id) { LocalResource entity = await _resourceRepository.Select.Where(a => a.Id == id).ToOneAsync(); ResourceDto resourceDto = _mapper.Map <ResourceDto>(entity); return(resourceDto); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS , FilePath scriptFileDir, FilePath processStartFile) { FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile ); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost" ).ToString(), 12345); URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new Path(scriptFile.GetAbsolutePath()))); LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>(); localResource.SetResource(localResourceUri); localResource.SetSize(-1); localResource.SetVisibility(LocalResourceVisibility.Application); localResource.SetType(LocalResourceType.File); localResource.SetTimestamp(scriptFile.LastModified()); string destinationFile = "dest_file"; IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource >(); localResources[destinationFile] = localResource; containerLaunchContext.SetLocalResources(localResources); IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile)); containerLaunchContext.SetCommands(commands); IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1" , 12345); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString ()); Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils .ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId (), nodeId, user), containerManagerBindAddress); currentUser.AddToken(nmToken); ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229 ()); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext ().GetContainerTokenSecretManager())); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); containerManager.StartContainers(allRequests); IList <ContainerId> containerIds = new AList <ContainerId>(); containerIds.AddItem(cId); GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds ); ContainerStatus containerStatus = containerManager.GetContainerStatuses(request). GetContainerStatuses()[0]; NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState( )); }
static List <Tuple <int, string, string> > GetGroupIdWithKeysAndShapeIds() { GroupService grpService = new GroupService(); BlobAccess blobAccess = new BlobAccess(); var container = blobAccess.LoadShapeFiles(); LocalResource myStorage = RoleEnvironment.GetLocalResource("LocalStorageWorkerRole"); List <Tuple <int, string, string> > GrpIdGroupKeyAndShapePaths = new List <Tuple <int, string, string> >(); GroupService.ParentGroup.ForEach(ParentGroup => { if (!String.IsNullOrWhiteSpace(ParentGroup.ShapeFileID) && ParentGroup.NotifySubgroups) // { string shapeIndex = LocalPath(myStorage, ParentGroup.ShapeFileID + ".shx"); string describeFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".dbf"); string shapeFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".shp"); string projectionFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".prj"); if (File.Exists(shapeIndex)) { File.Delete(shapeIndex); } if (File.Exists(describeFile)) { File.Delete(describeFile); } if (File.Exists(shapeFile)) { File.Delete(shapeFile); } if (File.Exists(projectionFile)) { File.Delete(projectionFile); } var ShxBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.IndexFile, ParentGroup.ShapeFileID)); ShxBlockBlobReference.DownloadToFile(shapeIndex, FileMode.Create); var DescribeBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.DescribeFile, ParentGroup.ShapeFileID)); DescribeBlockBlobReference.DownloadToFile(describeFile, FileMode.Create); var ShapeBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.ShapeFile, ParentGroup.ShapeFileID)); ShapeBlockBlobReference.DownloadToFile(shapeFile, FileMode.Create); var BlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.ProjectionFile, ParentGroup.ShapeFileID)); BlockBlobReference.DownloadToFile(projectionFile, FileMode.Create); GrpIdGroupKeyAndShapePaths.Add(new Tuple <int, string, string>(ParentGroup.GroupID, ParentGroup.SubGroupIdentificationKey, Path.Combine(myStorage.RootPath, ParentGroup.ShapeFileID))); } }); return(GrpIdGroupKeyAndShapePaths); }
private static DirectoryConfiguration CreateDirectoryConfiguration(LocalResource localResource) { DirectoryConfiguration dirConfig = new DirectoryConfiguration(); dirConfig.Container = "diagnostics-customlogfiles-container"; dirConfig.DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes; dirConfig.Path = localResource.RootPath; return(dirConfig); }
public CloudDriveManager(CloudStorageAccount account, string vhdPathAndName, char driveLetter, LocalResource cache) { _account = account; _vhdName = vhdPathAndName; _driveLetter = driveLetter; _cacheSize = cache.MaximumSizeInMegabytes / 2; _cache = cache; _logger = new Logger(account); }
private static LocalResource CreateLocalResource(DiagnosticMonitorConfiguration diagMonitorConfig) { LocalResource localResource = RoleEnvironment.GetLocalResource("CustomLogFiles"); DirectoryConfiguration dirConfig = CreateDirectoryConfiguration(localResource); diagMonitorConfig.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(1.0); diagMonitorConfig.Directories.DataSources.Add(dirConfig); return(localResource); }
/// <exception cref="System.IO.IOException"/> private SharedCacheUploader CreateSpiedUploader(LocalResource resource, Path localPath , string user, Configuration conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs) { SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user, conf, scmClient, fs, localFs); return(Org.Mockito.Mockito.Spy(uploader)); }
public BlockStoreCache(long maxHeap, string localStorageKey) { _maxBlockCount = maxHeap/BlockInfoSize; _cache = new ConcurrentDictionary<string, BlockInfo>(); _localStorage = RoleEnvironment.GetLocalResource(localStorageKey); int capacity = (_localStorage.MaximumSizeInMegabytes/4) - 10; // Just a bit of a safety buffer _diskCache = new BlockStoreDiskCache(_localStorage.RootPath + Path.DirectorySeparatorChar + "bscache", capacity, AzureBlockStore.AzureBlockSize); }
public static void Initialize() { //First get a reference to the local file structure. localStorage = RoleEnvironment.GetLocalResource("scratchpad"); var acc = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", RoleEnvironment.GetConfigurationSettingValue("StorageName"), RoleEnvironment.GetConfigurationSettingValue("StorageKey")); storageAccount = CloudStorageAccount.Parse(acc); blobClient = storageAccount.CreateCloudBlobClient(); }
private void CopyFromBlobToLocal(CloudBlobContainer blobContainer, LocalResource localStorage, string blobName) { CloudBlob node = blobContainer.GetBlobReference(blobName); using (BlobStream nodestream = node.OpenRead()) { var nodefile = Path.Combine(localStorage.RootPath, blobName); if (!System.IO.File.Exists(nodefile)) { using (var fileStream = new FileStream(nodefile, FileMode.CreateNew)) { nodestream.CopyTo(fileStream); fileStream.Flush(); fileStream.Close(); } } } }
public static void Initialize() { //First get a reference to the local file structure. localStorage = RoleEnvironment.GetLocalResource("scratchpad"); //Set up the Storage Account and Client #if DEBUG var acc = "UseDevelopmentStorage=true"; #else //points to microsoft azure var acc = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", RoleEnvironment.GetConfigurationSettingValue("StorageName"), RoleEnvironment.GetConfigurationSettingValue("AccountKey")); #endif storageAccount = CloudStorageAccount.Parse(acc); blobClient = storageAccount.CreateCloudBlobClient(); }
private static void DoInitCloudDriveCache(LocalResource cloudDriveCacheLocalResource) { const int tries = 30; // Temporary workaround for ERROR_UNSUPPORTED_OS seen with Windows Azure Drives // See http://blogs.msdn.com/b/windowsazurestorage/archive/2010/12/17/error-unsupported-os-seen-with-windows-azure-drives.aspx for (int i = 0; i < tries; i++) try { CloudDrive.InitializeCache(cloudDriveCacheLocalResource.RootPath, cloudDriveCacheLocalResource.MaximumSizeInMegabytes); break; } catch (CloudDriveException ex) { if (!ex.Message.Equals("ERROR_UNSUPPORTED_OS")) throw; if (i >= (tries - 1)) { // If the workaround fails then it would be dangerous to continue silently, so exit Log.Error( "Workaround for ERROR_UNSUPPORTED_OS see http://bit.ly/fw7qzo FAILED"); Environment.Exit(-1); } Log.Info("Using temporary workaround for ERROR_UNSUPPORTED_OS see http://bit.ly/fw7qzo"); Thread.Sleep(10000); } }
public WorkerRoleIO() { this._LocalResource = RoleEnvironment.GetLocalResource("Regisfra.LocalStorage"); }
private CloudDriveSettings() { BlobContainerName = RoleEnvironment.GetConfigurationSettingValue(CloudDriveBlobContainerConfigOption); BlobName = RoleEnvironment.GetConfigurationSettingValue(CloudDriveBlobNameConfigOption); BlobSize = ushort.Parse(RoleEnvironment.GetConfigurationSettingValue(CloudDriveBlobSizeConfigOption)); ConnectionString = RoleEnvironment.GetConfigurationSettingValue(CloudDriveConnectionStringConfigOption); InitCache = bool.Parse(RoleEnvironment.GetConfigurationSettingValue(CloudDriveInitCacheConfigOption)); CacheLocalResource = null; if (InitCache) { var cacheResourceName = RoleEnvironment.GetConfigurationSettingValue(CloudDriveCacheLocalResourceNameConfigOption); CacheLocalResource = RoleEnvironment.GetLocalResource(cacheResourceName); } }
private void OnInitialize() { var selfInstance = InstanceEnumerator.EnumerateInstances().First(i => i.IsSelf); cloudStorageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue(ConfigurationSettingsKeys.StorageConnectionString)); log.Info("Storage account selected: {0}",cloudStorageAccount.BlobEndpoint); cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient(); log.Info("Storage client created"); var containerName = ConfigurationProvider.GetSetting(ConfigurationSettingsKeys.StorageContainerName,"ravendb"); // In order to force a connection we just enumerate all available containers: var availableContainers = cloudBlobClient.ListContainers().ToArray(); foreach (var container in availableContainers) { log.Info("Available container: {0}",container.Name); } if (!availableContainers.Any(c => c.Name.Equals(containerName))) { log.Info("Container {0} does not exist, creating",containerName); // Container does not exist: cloudBlobClient.GetContainerReference(containerName).Create(); } cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName); log.Info("Container {0} selected",cloudBlobContainer.Name); localCache = RoleEnvironment.GetLocalResource(ConfigurationSettingsKeys.StorageCacheResource); log.Info("Cache resource retrieved: {0}, path: {1}",localCache.Name,localCache.RootPath); CloudDrive.InitializeCache(localCache.RootPath, localCache.MaximumSizeInMegabytes); log.Info("Cache initialized: {0} mb",localCache.MaximumSizeInMegabytes); var driveName = string.Format("{0}{1}.vhd", selfInstance.InstanceType, selfInstance.InstanceIndex).ToLowerInvariant(); log.Info("Virtual drive name: {0}",driveName); var pageBlob = cloudBlobContainer.GetPageBlobReference(driveName); log.Info("Virtual drive blob: {0}",pageBlob.Uri); cloudDrive = cloudStorageAccount.CreateCloudDrive(pageBlob.Uri.ToString()); log.Info("Virtual drive created: {0}",cloudDrive.Uri); var storageSize = ConfigurationProvider.GetSetting(ConfigurationSettingsKeys.StorageSize, 50000); log.Info("Storage size: {0} mb",storageSize); cloudDrive.CreateIfNotExist(storageSize); log.Info("Virtual drive initialized: {0}",cloudDrive.Uri); var mountedDirectoryPath = cloudDrive.Mount(storageSize, DriveMountOptions.None); log.Info("Virtual drive mounted at: {0}",mountedDirectoryPath); mountedDirectory = new DirectoryInfo(mountedDirectoryPath); log.Info("Ensuring drive is available: {0}",mountedDirectoryPath); UpdateTestFile(); log.Info("Storage initialization succeeded"); }
private void InitializeLocalCache() { _localCache = RoleEnvironment.GetLocalResource("RavenCache"); CloudDrive.InitializeCache(_localCache.RootPath.TrimEnd('\\'), _localCache.MaximumSizeInMegabytes); }
// Use this for initialization protected virtual void Start() { dependencyLines = new Dictionary<GameObject, VectorLine>(); defaultColor = renderer.material.color; if(transform.parent != null) { planet = transform.parent.GetComponent<Planet>(); if(planet != null) { //if(!CheckRequirements(planet)) // Destroy(gameObject); } } //maintain global scale if(transform.parent != null) { Vector3 correctedScale = new Vector3(transform.localScale.x / transform.parent.localScale.x, transform.localScale.y / transform.parent.localScale.y, transform.localScale.z / transform.parent.localScale.z); transform.localScale = correctedScale; } //Initialize input = new LocalResource[0]; output = new LocalResource(); output.type = LocalResourceType.Power; output.quantity = 5; outputRange = DEFAULT_OUTPUT_RANGE; }
public HostedAzureLocalResource(LocalResource localResource) { _localResource = localResource; }