public string UploadFolder(string containerName, string localStoragePath, string prefixAzureDirectoryName)
{
try {
LocalResource Storage = RoleEnvironment.GetLocalResource("ZYLocalStorage");
string[] filePaths = Directory.GetFiles(localStoragePath);
foreach (var item in filePaths)
{
if (!Path.GetExtension(item).Equals(".zip"))
{
UploadFile(item, Path.GetFileName(item), prefixAzureDirectoryName);
}
}
var folder = new DirectoryInfo(localStoragePath);
var subFolders = folder.GetDirectories();
foreach (var directoryInfo in subFolders)
{
UploadFolder("zycontainer", Storage.RootPath + directoryInfo.Name, prefixAzureDirectoryName + "/" + directoryInfo.Name);
}
return("Upload Directory seccussfully.");
}
catch (Exception e)
{ return(e.Message); }
}
private void MountSnapshot(string snapshotUri)
{
string storageAccountName = txtStorageAccountName.Text;
string storageAccountKey = txtStorageAccountKey.Text;
try
{
LocalResource resource = WindowsAzureSystemHelper.GetLocalStorageResource(LOCAL_STORAGE_NAME);
int cacheSize = int.Parse(txtCacheSize.Text);
if (resource.MaximumSizeInMegabytes < cacheSize)
{
throw new Exception(String.Format("Cache size {0} MB cannot be more than maximum permitted size {1} MB", cacheSize, resource.MaximumSizeInMegabytes));
}
bool wasAlreadyMounted;
string driveName = WADriveHelper.MountAzureDrive(storageAccountName, storageAccountKey, new Uri(snapshotUri), LocalStoragePath + txtCacheDirName.Text + ".snapshot",
int.Parse(txtCacheSize.Text),
int.Parse(txtDriveSize.Text),
DriveMountOptions.None, out wasAlreadyMounted);
lblMsg.Text = String.Format("Mounted Snapshot {0} to Drive {1}", snapshotUri, driveName);
ListAllMountedDrives();
}
catch (Exception ex)
{
lblMsg.Text = ex.Message;
}
}
public static void Initialize()
{
//First get a reference to the local file structure.
localStorage = RoleEnvironment.GetLocalResource("scratchpad");
//Check to see if we have already grabbed the component files.
if (!CloudBackedStore.DirectoryExists(ComponetDir))
{
var url = string.Format("http://{0}.blob.core.windows.net/components/", RoleEnvironment.GetConfigurationSettingValue("StorageName"));
foreach (var app in OdpiAppRepo.Apps)
{
if( !string.IsNullOrEmpty(app.PackageName) )
CloudBackedStore.Grab(ComponetDir, "\\" + app.PackageName, "components", url + app.PackageName);
if (!string.IsNullOrEmpty(app.ConfName))
CloudBackedStore.Grab(ComponetDir, "\\" + app.ConfName, "components", url + app.ConfName);
var req = app.RequiredFiles;
if (req != null)
{
foreach (var file in req)
{
CloudBackedStore.Grab(ComponetDir, "\\" + file, "components", url + file);
}
}
}
}
}
public async Task <IEnumerable <LocalResource> > SearchPhoneNumbersAsync(string areaCode = "415")
{
var localNumbers = await LocalResource.ReadAsync(
pathCountryCode : "US", areaCode : int.Parse(areaCode), client : _client);
return(localNumbers.ToList());
}
//Download many blobs in one folder of container to local storage
public void DownloadToLocalStorage(LocalResource mystorage, string folderNameInContainer, string fileRootPath)
{
string blobPrefix = folderNameInContainer + "/";
bool useFlatBlobListing = false;
var blobs = Container.ListBlobs(blobPrefix, useFlatBlobListing, BlobListingDetails.None);
var filesOrFoders = blobs.ToList();
//Create direcotry in local storage
Directory.CreateDirectory(fileRootPath);
string fileOrFolderName;
//If it is file, download it. If it is directory, use this function again and add this folder name after original folder name
foreach(var fileOrFolder in filesOrFoders)
{
if(fileOrFolder.GetType() != typeof(CloudBlobDirectory))
{
fileOrFolderName = Path.GetFileName(fileOrFolder.Uri.ToString());
DownloadBlob(Path.Combine(folderNameInContainer, fileOrFolderName), Path.Combine(fileRootPath, fileOrFolderName));
}
else if(fileOrFolder.GetType() == typeof(CloudBlobDirectory))
{
fileOrFolderName = fileOrFolder.Uri.ToString();
fileOrFolderName = fileOrFolderName.Substring(0, fileOrFolderName.Length - 1);
fileOrFolderName = fileOrFolderName.Substring(fileOrFolderName.LastIndexOf("/") + 1);
//Call this function again
DownloadToLocalStorage(mystorage, folderNameInContainer + "/" + fileOrFolderName, Path.Combine(fileRootPath, fileOrFolderName));
}
}
}
internal Task <IDiscoveryResource> PublishAsync(
string category,
string connection,
int expirySeconds,
IReadOnlyDictionary <string, string> attributes = null,
CancellationToken token = default)
{
var attrs = new Dictionary <string, string>();
if (attributes != null) // copy so user can't change them behind our back
{
foreach (var kvp in attributes)
{
attrs[kvp.Key] = kvp.Value;
}
}
var resource = new LocalResource(category, connection, expirySeconds, attrs);
resource.Updated = OnResourceUpdated;
lock (this)
{
localResources_.Add(resource); // new local resources get a new guid, always unique
UpdateAnnounceTimer();
}
return(Task.FromResult((IDiscoveryResource)resource));
}
SKBitmap GetCoverBitmap()
{
using (Stream stream = LocalResource.GetStream("cover.png", this))
{
return(SKBitmap.Decode(stream));
};
}
public Worker(string name)
{
Name = name;
LocalResource azureLocalResource = RoleEnvironment.GetLocalResource("LocalStorage");
WorkDirectoryName = Path.Combine(azureLocalResource.RootPath, "WD" + name);
/*** Init ***/
// On instancie un objet de chaque classe héritant de BusinessProcess
_employees = new List <IBusinessProcess>();
Object[] args = new object[] { WorkDirectoryName };
foreach (Type instance in
typeof(BusinessProcess).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BusinessProcess))))
{
_employees.Add(Activator.CreateInstance(instance, args) as IBusinessProcess);
}
//// TODO : find a good time
int maximumTimeToSleepInSeconde;
if (!int.TryParse(RoleEnvironment.GetConfigurationSettingValue("MaximumSecondesTimeSleep"),
out maximumTimeToSleepInSeconde))
{
maximumTimeToSleepInSeconde = 2 * 60 * 60; // 2 heures
}
CustomSleeper = new Sleeper(2, maximumTimeToSleepInSeconde);
CountLoop = 0;
}
public IncomingPhoneNumberResource Hop(
string twilioAccountSid,
string relaySid,
ILogging logging)
{
var original = IncomingPhoneNumberResource.Fetch(relaySid);
string relayPhoneNumber = original.PhoneNumber.ToString();
logging.Log($"Releasing phone number {relayPhoneNumber} {relaySid}");
IncomingPhoneNumberResource.Delete(relaySid);
var localAvailableNumbers = LocalResource.Read("US");
var desiredNumber = localAvailableNumbers.First();
logging.Log($"Purchasing new phone number {desiredNumber.PhoneNumber}");
var newNumber = IncomingPhoneNumberResource.Create(
twilioAccountSid,
desiredNumber.PhoneNumber,
smsMethod: original.SmsMethod,
smsUrl: original.SmsUrl);
logging.Log($"New phone number purchased {newNumber.PhoneNumber} {newNumber.Sid}");
return(newNumber);
}
public static void Initialize()
{
//First get a reference to the local file structure.
localStorage = RoleEnvironment.GetLocalResource("scratchpad");
//Check to see if we have already grabbed the component files.
if (!CloudBackedStore.DirectoryExists(ComponetDir))
{
var url = string.Format("http://{0}.blob.core.windows.net/components/", RoleEnvironment.GetConfigurationSettingValue("StorageName"));
foreach (var app in OdpiAppRepo.Apps)
{
if (!string.IsNullOrEmpty(app.PackageName))
{
CloudBackedStore.Grab(ComponetDir, "\\" + app.PackageName, "components", url + app.PackageName);
}
if (!string.IsNullOrEmpty(app.ConfName))
{
CloudBackedStore.Grab(ComponetDir, "\\" + app.ConfName, "components", url + app.ConfName);
}
var req = app.RequiredFiles;
if (req != null)
{
foreach (var file in req)
{
CloudBackedStore.Grab(ComponetDir, "\\" + file, "components", url + file);
}
}
}
}
}
public async Task tryPutResourceIn <T>(string url, Action <byte[]> loadAction)
{
bool isCached;
string localpath;
// var result = await Task.Run(() => tryReadFileFromLocalCache(url, out isCached, out localpath));
LocalResource localResource = await tryReadFileFromLocalCache(url);
var result = localResource.Bytes;
isCached = localResource.InCache;
localpath = localResource.LocalPath;
if (!isCached)
{
try
{
var request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
await makeRequest(request, (stream, lpath) => { var s = new MemoryStream(); stream.CopyTo(s); result = s.ToArray(); }, err => Debug.WriteLine(err.ToString()), true);
}
catch
{
result = null;
}
}
loadAction(result);
}
public static String GetLocalStoragePath()
{
//returns the full path of the local storage
LocalResource l = RoleEnvironment.GetLocalResource("LocalSoundStore");
return(string.Format(l.RootPath));
}
public virtual void TestGetActualPath()
{
Configuration conf = new Configuration();
conf.SetBoolean(YarnConfiguration.SharedCacheEnabled, true);
LocalResource resource = Org.Mockito.Mockito.Mock <LocalResource>();
// give public visibility
Org.Mockito.Mockito.When(resource.GetVisibility()).ThenReturn(LocalResourceVisibility
.Public);
Path localPath = new Path("foo.jar");
string user = "******";
SCMUploaderProtocol scmClient = Org.Mockito.Mockito.Mock <SCMUploaderProtocol>();
FileSystem fs = Org.Mockito.Mockito.Mock <FileSystem>();
FileSystem localFs = Org.Mockito.Mockito.Mock <FileSystem>();
// stub it to return a status that indicates a directory
FileStatus status = Org.Mockito.Mockito.Mock <FileStatus>();
Org.Mockito.Mockito.When(status.IsDirectory()).ThenReturn(true);
Org.Mockito.Mockito.When(localFs.GetFileStatus(localPath)).ThenReturn(status);
SharedCacheUploader spied = CreateSpiedUploader(resource, localPath, user, conf,
scmClient, fs, localFs);
Path actualPath = spied.GetActualPath();
NUnit.Framework.Assert.AreEqual(actualPath.GetName(), localPath.GetName());
NUnit.Framework.Assert.AreEqual(actualPath.GetParent().GetName(), localPath.GetName
());
}
async void OnSourceChanged()
{
if (Source.Contains("http"))
{
using (Stream stream = await httpClient.GetStreamAsync(Source))
{
using (MemoryStream memStream = new MemoryStream())
{
await stream.CopyToAsync(memStream);
memStream.Seek(0, SeekOrigin.Begin);
skSvg.Load(memStream);
}
}
}
else
{
using (var memStream = LocalResource.GetStream(Source))
{
memStream.Seek(0, SeekOrigin.Begin);
skSvg.Load(memStream);
}
}
InvalidateSurface();
}
void Application_Start(object sender, EventArgs e)
{
// Code that runs on application startup
BundleConfig.RegisterBundles(BundleTable.Bundles);
AuthConfig.RegisterOpenAuth();
if (imageStorePath == null)
{
ImageStorePath = WebConfigurationManager.AppSettings["ImageStorePath"];
}
// initialize storage account configuration setting publisher
CloudStorageAccount.SetConfigurationSettingPublisher((configName, configSetter) =>
{
string connectionString = RoleEnvironment.GetConfigurationSettingValue(configName);
configSetter(connectionString);
});
try
{
// initialize the local cache for the Azure drive
LocalResource cache = RoleEnvironment.GetLocalResource("LocalDriveCache");
CloudDrive.InitializeCache(cache.RootPath + "cache", cache.MaximumSizeInMegabytes);
// retrieve storage account
CloudStorageAccount account = CloudStorageAccount.FromConfigurationSetting("DataConnectionString");
// retrieve URI for the page blob that contains the cloud drive from configuration settings
string imageStoreBlobUri = RoleEnvironment.GetConfigurationSettingValue("ImageStoreBlobUri");
// unmount any previously mounted drive.
foreach (var drive in CloudDrive.GetMountedDrives())
{
var mountedDrive = new CloudDrive(drive.Value, account.Credentials);
mountedDrive.Unmount();
}
// create the Windows Azure drive and its associated page blob
CloudDrive imageStoreDrive = account.CreateCloudDrive(imageStoreBlobUri);
if (CloudDrive.GetMountedDrives().Count == 0)
{
try
{
imageStoreDrive.Create(16);
}
catch (CloudDriveException)
{
// drive already exists
}
}
// mount the drive and initialize the application with the path to the image store on the Azure drive
Global.ImageStorePath = imageStoreDrive.Mount(cache.MaximumSizeInMegabytes / 2, DriveMountOptions.None);
}
catch (CloudDriveException driveException)
{
Trace.WriteLine("Error: " + driveException.Message);
}
}
/// <exception cref="System.IO.IOException"/>
private void AddToLocalResources(FileSystem fs, string fileSrcPath, string fileDstPath
, string appId, IDictionary <string, LocalResource> localResources, string resources
)
{
string suffix = appName + "/" + appId + "/" + fileDstPath;
Path dst = new Path(fs.GetHomeDirectory(), suffix);
if (fileSrcPath == null)
{
FSDataOutputStream ostream = null;
try
{
ostream = FileSystem.Create(fs, dst, new FsPermission((short)0x1c8));
ostream.WriteUTF(resources);
}
finally
{
IOUtils.CloseQuietly(ostream);
}
}
else
{
fs.CopyFromLocalFile(new Path(fileSrcPath), dst);
}
FileStatus scFileStatus = fs.GetFileStatus(dst);
LocalResource scRsrc = LocalResource.NewInstance(ConverterUtils.GetYarnUrlFromURI
(dst.ToUri()), LocalResourceType.File, LocalResourceVisibility.Application, scFileStatus
.GetLen(), scFileStatus.GetModificationTime());
localResources[fileDstPath] = scRsrc;
}
async Task LocalFetch()
{
await Task.Delay(1000);
var content = LocalResource.GetContent("product-by-category.json");
var listProductByCategory = JsonSerializer.Deserialize <ProductByCategory[]>(content, new JsonSerializerOptions {
PropertyNameCaseInsensitive = true
});
var _categories = new List <CategoryModel> {
new() { Name = "All" }
};
_categories.AddRange(listProductByCategory.Select(_ => new CategoryModel {
Name = _.Category
}));
categories = _categories.ToArray();
products = listProductByCategory
.SelectMany(_ => _.Products
.Select(product => new ProductModel
{
Title = product.Title,
Summary = product.Summary,
Description = product.Description,
Price = $"${product.Price:N2}",
UrlImage = product.UrlImage,
Order = product.Order,
Category = _.Category,
CategoryTag = _.Category.ToUpper(),
}))
.OrderBy(_ => _.Order)
.ToArray();
}
public void RemoveObserver(String clientID, LocalResource resource)
{
IDictionary <String, Observationship> resourceObservers = null;
IDictionary <String, Observationship> clientObservers = null;
String path = resource.Path;
if (_observersByResource.ContainsKey(path))
{
resourceObservers = _observersByResource[path];
}
if (_observersByClient.ContainsKey(clientID))
{
clientObservers = _observersByClient[clientID];
}
if (resourceObservers != null && clientObservers != null)
{
lock (_sync)
{
if (resourceObservers.Remove(clientID) && clientObservers.Remove(path))
{
if (log.IsInfoEnabled)
{
log.Info(String.Format("Terminated observing relationship by GET: {0} @ {1}", clientID, path));
}
return;
}
}
}
if (log.IsWarnEnabled)
{
log.Warn(String.Format("Cannot find observing relationship: {0} @ {1}", clientID, path));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestSetupDistributedCacheConflictsFiles()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem
));
URI mockUri = URI.Create("mockfs://mock/");
FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem
();
URI file = new URI("mockfs://mock/tmp/something.zip#something");
Path filePath = new Path(file);
URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
Path file2Path = new Path(file2);
Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath);
Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path);
DistributedCache.AddCacheFile(file, conf);
DistributedCache.AddCacheFile(file2, conf);
conf.Set(MRJobConfig.CacheFileTimestamps, "10,11");
conf.Set(MRJobConfig.CacheFilesSizes, "10,11");
conf.Set(MRJobConfig.CacheFileVisibilities, "true,true");
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
NUnit.Framework.Assert.AreEqual(1, localResources.Count);
LocalResource lr = localResources["something"];
//First one wins
NUnit.Framework.Assert.IsNotNull(lr);
NUnit.Framework.Assert.AreEqual(10l, lr.GetSize());
NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp());
NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
internal static LocalResource CreateZipFile(FileContext files, Path p, int len, Random
r, LocalResourceVisibility vis)
{
byte[] bytes = new byte[len];
r.NextBytes(bytes);
FilePath archiveFile = new FilePath(p.ToUri().GetPath() + ".ZIP");
archiveFile.CreateNewFile();
ZipOutputStream @out = new ZipOutputStream(new FileOutputStream(archiveFile));
@out.PutNextEntry(new ZipEntry(p.GetName()));
@out.Write(bytes);
@out.CloseEntry();
@out.Close();
LocalResource ret = recordFactory.NewRecordInstance <LocalResource>();
ret.SetResource(ConverterUtils.GetYarnUrlFromPath(new Path(p.ToString() + ".ZIP")
));
ret.SetSize(len);
ret.SetType(LocalResourceType.Archive);
ret.SetVisibility(vis);
ret.SetTimestamp(files.GetFileStatus(new Path(p.ToString() + ".ZIP")).GetModificationTime
());
return(ret);
}
/// <exception cref="System.Exception"/>
public virtual void TestUniqueDestinationPath()
{
Configuration conf = new Configuration();
FileContext files = FileContext.GetLocalFSFileContext(conf);
Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name
));
files.Mkdir(basedir, null, true);
conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString());
ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor();
LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName);
Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf);
destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
Path p = new Path(basedir, "dir" + 0 + ".jar");
LocalResourceVisibility vis = LocalResourceVisibility.Private;
LocalResource rsrc = CreateJar(files, p, vis);
FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf
, destPath, rsrc);
Future <Path> rPath = singleThreadedExec.Submit(fsd);
singleThreadedExec.Shutdown();
while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds))
{
}
NUnit.Framework.Assert.IsTrue(rPath.IsDone());
// Now FSDownload will not create a random directory to localize the
// resource. Therefore the final localizedPath for the resource should be
// destination directory (passed as an argument) + file name.
NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent());
}
public static string Mount()
{
CloudStorageAccount storageAccount;
if (RoleEnvironment.IsEmulated)
{
storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
}
else
{
storageAccount = CloudStorageAccount.Parse(connectionString);
}
LocalResource localCache = RoleEnvironment.GetLocalResource("InstanceDriveCache");
CloudDrive.InitializeCache(localCache.RootPath, localCache.MaximumSizeInMegabytes);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
blobClient.GetContainerReference("drives").CreateIfNotExist();
myCloudDrive = storageAccount.CreateCloudDrive(
blobClient
.GetContainerReference("drives")
.GetPageBlobReference("mysupercooldrive.vhd")
.Uri.ToString()
);
myCloudDrive.CreateIfNotExist(64);
return(myCloudDrive.Mount(25, DriveMountOptions.None));
}
protected override void DoExecute(Request request)
{
if (request != null)
{
String path = request.UriPath;
LocalResource resource = GetResource(path);
if (resource != null)
{
request.Resource = resource;
if (log.IsDebugEnabled)
{
log.Debug("Dispatching execution: " + path);
}
_dispatcher.Dispatch(request);
}
else if (request.Code == Code.PUT)
{
// allows creation of non-existing resources through PUT
CreateByPut(path, request);
request.SendResponse();
}
else
{
if (log.IsWarnEnabled)
{
log.Warn("Cannot find resource: " + path);
}
request.Respond(Code.NotFound);
request.SendResponse();
}
}
}
public static async Task <ResourceSet <LocalResource> > GetAvailableLocalPhoneNumberAsync(ITwilioRestClient client, CountryCode countryCode, string accountSid = null, int?areaCode = null, bool?smsEnabled = null, bool?mmsEnabled = null, bool?faxEnabled = null, bool?voiceEnabled = null, bool?beta = null, string containsPattern = null, string lata = null, string latLong = null, string nearNumber = null, int?distance = null, string locality = null, string postalCode = null, string rateCenter = null, string region = null, bool?excludeAllAddressRequired = null, bool?excludeForeignAddressRequired = null, bool?excludeLocalAddressRequired = null, long?limit = null)
{
var nearPhoneNumber = nearNumber != null ? new PhoneNumber(nearNumber) : null;
var options = new ReadLocalOptions($"{countryCode:G}")
{
PathAccountSid = accountSid,
AreaCode = areaCode,
SmsEnabled = smsEnabled,
MmsEnabled = mmsEnabled,
FaxEnabled = faxEnabled,
VoiceEnabled = voiceEnabled,
Beta = beta,
Contains = containsPattern,
InLata = lata,
NearLatLong = latLong,
NearNumber = nearPhoneNumber,
Distance = distance,
InLocality = locality,
InPostalCode = postalCode,
InRateCenter = rateCenter,
InRegion = region,
ExcludeAllAddressRequired = excludeAllAddressRequired,
ExcludeForeignAddressRequired = excludeForeignAddressRequired,
ExcludeLocalAddressRequired = excludeLocalAddressRequired,
Limit = limit,
PageSize = null
};
return(await LocalResource.ReadAsync(options, client));
}
public void AddObserver(Request request, LocalResource resource)
{
request.IsObserving = true;
Observationship shipToAdd = new Observationship(request);
lock (_sync)
{
// get clients map for the given resource path
IDictionary <String, Observationship> resourceObservers = SafeGet(_observersByResource, resource.Path);
// get resource map for given client address
IDictionary <String, Observationship> clientObservers = SafeGet(_observersByClient, request.PeerAddress.ToString());
// save relationship for notifications triggered by resource
resourceObservers[request.PeerAddress.ToString()] = shipToAdd;
// save relationship for actions triggered by client
clientObservers[resource.Path] = shipToAdd;
}
if (log.IsInfoEnabled)
{
log.Info(String.Format("Established observing relationship: {0} @ {1}",
request.PeerAddress, resource.Path));
}
// update response
PrepareResponse(request);
}
private static DirectoryConfiguration CreateDirectoryConfiguration(LocalResource localResource)
{
DirectoryConfiguration dirConfig = new DirectoryConfiguration();
dirConfig.Container = "diagnostics-customlogfiles-container";
dirConfig.DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes;
dirConfig.Path = localResource.RootPath;
return dirConfig;
}
internal MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag, URL
localPath, SerializedException ex)
{
this.rsrc = rsrc;
this.tag = tag;
this.localPath = localPath;
this.ex = ex;
}
public void Dispose()
{
if (Directory.Exists(this.CurrentDirectory))
Directory.Delete(this.CurrentDirectory, true);
this.CurrentDirectory = null;
this._LocalResource = null;
}
public async Task <ResourceDto> GetAsync(long id)
{
LocalResource entity = await _resourceRepository.Select.Where(a => a.Id == id).ToOneAsync();
ResourceDto resourceDto = _mapper.Map <ResourceDto>(entity);
return(resourceDto);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public static void StartContainer(NodeManager nm, ContainerId cId, FileContext localFS
, FilePath scriptFileDir, FilePath processStartFile)
{
FilePath scriptFile = CreateUnhaltingScriptFile(cId, scriptFileDir, processStartFile
);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
NodeId nodeId = BuilderUtils.NewNodeId(Sharpen.Extensions.GetAddressByName("localhost"
).ToString(), 12345);
URL localResourceUri = ConverterUtils.GetYarnUrlFromPath(localFS.MakeQualified(new
Path(scriptFile.GetAbsolutePath())));
LocalResource localResource = recordFactory.NewRecordInstance <LocalResource>();
localResource.SetResource(localResourceUri);
localResource.SetSize(-1);
localResource.SetVisibility(LocalResourceVisibility.Application);
localResource.SetType(LocalResourceType.File);
localResource.SetTimestamp(scriptFile.LastModified());
string destinationFile = "dest_file";
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
localResources[destinationFile] = localResource;
containerLaunchContext.SetLocalResources(localResources);
IList <string> commands = Arrays.AsList(Shell.GetRunScriptCommand(scriptFile));
containerLaunchContext.SetCommands(commands);
IPEndPoint containerManagerBindAddress = NetUtils.CreateSocketAddrForHost("127.0.0.1"
, 12345);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(cId.ToString
());
Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> nmToken = ConverterUtils
.ConvertFromYarn(nm.GetNMContext().GetNMTokenSecretManager().CreateNMToken(cId.GetApplicationAttemptId
(), nodeId, user), containerManagerBindAddress);
currentUser.AddToken(nmToken);
ContainerManagementProtocol containerManager = currentUser.DoAs(new _PrivilegedAction_229
());
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, TestContainerManager.CreateContainerToken(cId, 0, nodeId, user, nm.GetNMContext
().GetContainerTokenSecretManager()));
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
containerManager.StartContainers(allRequests);
IList <ContainerId> containerIds = new AList <ContainerId>();
containerIds.AddItem(cId);
GetContainerStatusesRequest request = GetContainerStatusesRequest.NewInstance(containerIds
);
ContainerStatus containerStatus = containerManager.GetContainerStatuses(request).
GetContainerStatuses()[0];
NUnit.Framework.Assert.AreEqual(ContainerState.Running, containerStatus.GetState(
));
}
static List <Tuple <int, string, string> > GetGroupIdWithKeysAndShapeIds()
{
GroupService grpService = new GroupService();
BlobAccess blobAccess = new BlobAccess();
var container = blobAccess.LoadShapeFiles();
LocalResource myStorage = RoleEnvironment.GetLocalResource("LocalStorageWorkerRole");
List <Tuple <int, string, string> > GrpIdGroupKeyAndShapePaths = new List <Tuple <int, string, string> >();
GroupService.ParentGroup.ForEach(ParentGroup =>
{
if (!String.IsNullOrWhiteSpace(ParentGroup.ShapeFileID) && ParentGroup.NotifySubgroups) //
{
string shapeIndex = LocalPath(myStorage, ParentGroup.ShapeFileID + ".shx");
string describeFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".dbf");
string shapeFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".shp");
string projectionFile = LocalPath(myStorage, ParentGroup.ShapeFileID + ".prj");
if (File.Exists(shapeIndex))
{
File.Delete(shapeIndex);
}
if (File.Exists(describeFile))
{
File.Delete(describeFile);
}
if (File.Exists(shapeFile))
{
File.Delete(shapeFile);
}
if (File.Exists(projectionFile))
{
File.Delete(projectionFile);
}
var ShxBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.IndexFile, ParentGroup.ShapeFileID));
ShxBlockBlobReference.DownloadToFile(shapeIndex, FileMode.Create);
var DescribeBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.DescribeFile, ParentGroup.ShapeFileID));
DescribeBlockBlobReference.DownloadToFile(describeFile, FileMode.Create);
var ShapeBlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.ShapeFile, ParentGroup.ShapeFileID));
ShapeBlockBlobReference.DownloadToFile(shapeFile, FileMode.Create);
var BlockBlobReference = container.GetBlockBlobReference(GetPath(FileType.ProjectionFile, ParentGroup.ShapeFileID));
BlockBlobReference.DownloadToFile(projectionFile, FileMode.Create);
GrpIdGroupKeyAndShapePaths.Add(new Tuple <int, string, string>(ParentGroup.GroupID, ParentGroup.SubGroupIdentificationKey, Path.Combine(myStorage.RootPath, ParentGroup.ShapeFileID)));
}
});
return(GrpIdGroupKeyAndShapePaths);
}
private static DirectoryConfiguration CreateDirectoryConfiguration(LocalResource localResource)
{
DirectoryConfiguration dirConfig = new DirectoryConfiguration();
dirConfig.Container = "diagnostics-customlogfiles-container";
dirConfig.DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes;
dirConfig.Path = localResource.RootPath;
return(dirConfig);
}
public CloudDriveManager(CloudStorageAccount account, string vhdPathAndName, char driveLetter, LocalResource cache)
{
_account = account;
_vhdName = vhdPathAndName;
_driveLetter = driveLetter;
_cacheSize = cache.MaximumSizeInMegabytes / 2;
_cache = cache;
_logger = new Logger(account);
}
private static LocalResource CreateLocalResource(DiagnosticMonitorConfiguration diagMonitorConfig)
{
LocalResource localResource = RoleEnvironment.GetLocalResource("CustomLogFiles");
DirectoryConfiguration dirConfig = CreateDirectoryConfiguration(localResource);
diagMonitorConfig.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(1.0);
diagMonitorConfig.Directories.DataSources.Add(dirConfig);
return(localResource);
}
/// <exception cref="System.IO.IOException"/>
private SharedCacheUploader CreateSpiedUploader(LocalResource resource, Path localPath
, string user, Configuration conf, SCMUploaderProtocol scmClient, FileSystem fs,
FileSystem localFs)
{
SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user,
conf, scmClient, fs, localFs);
return(Org.Mockito.Mockito.Spy(uploader));
}
public BlockStoreCache(long maxHeap, string localStorageKey)
{
_maxBlockCount = maxHeap/BlockInfoSize;
_cache = new ConcurrentDictionary<string, BlockInfo>();
_localStorage = RoleEnvironment.GetLocalResource(localStorageKey);
int capacity = (_localStorage.MaximumSizeInMegabytes/4) - 10; // Just a bit of a safety buffer
_diskCache = new BlockStoreDiskCache(_localStorage.RootPath + Path.DirectorySeparatorChar + "bscache",
capacity, AzureBlockStore.AzureBlockSize);
}
public static void Initialize()
{
//First get a reference to the local file structure.
localStorage = RoleEnvironment.GetLocalResource("scratchpad");
var acc = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", RoleEnvironment.GetConfigurationSettingValue("StorageName"), RoleEnvironment.GetConfigurationSettingValue("StorageKey"));
storageAccount = CloudStorageAccount.Parse(acc);
blobClient = storageAccount.CreateCloudBlobClient();
}
private void CopyFromBlobToLocal(CloudBlobContainer blobContainer, LocalResource localStorage, string blobName)
{
CloudBlob node = blobContainer.GetBlobReference(blobName);
using (BlobStream nodestream = node.OpenRead()) {
var nodefile = Path.Combine(localStorage.RootPath, blobName);
if (!System.IO.File.Exists(nodefile)) {
using (var fileStream = new FileStream(nodefile, FileMode.CreateNew)) {
nodestream.CopyTo(fileStream);
fileStream.Flush();
fileStream.Close();
}
}
}
}
public static void Initialize()
{
//First get a reference to the local file structure.
localStorage = RoleEnvironment.GetLocalResource("scratchpad");
//Set up the Storage Account and Client
#if DEBUG
var acc = "UseDevelopmentStorage=true";
#else
//points to microsoft azure
var acc = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", RoleEnvironment.GetConfigurationSettingValue("StorageName"), RoleEnvironment.GetConfigurationSettingValue("AccountKey"));
#endif
storageAccount = CloudStorageAccount.Parse(acc);
blobClient = storageAccount.CreateCloudBlobClient();
}
private static void DoInitCloudDriveCache(LocalResource cloudDriveCacheLocalResource)
{
const int tries = 30;
// Temporary workaround for ERROR_UNSUPPORTED_OS seen with Windows Azure Drives
// See http://blogs.msdn.com/b/windowsazurestorage/archive/2010/12/17/error-unsupported-os-seen-with-windows-azure-drives.aspx
for (int i = 0; i < tries; i++)
try
{
CloudDrive.InitializeCache(cloudDriveCacheLocalResource.RootPath, cloudDriveCacheLocalResource.MaximumSizeInMegabytes);
break;
}
catch (CloudDriveException ex)
{
if (!ex.Message.Equals("ERROR_UNSUPPORTED_OS"))
throw;
if (i >= (tries - 1))
{
// If the workaround fails then it would be dangerous to continue silently, so exit
Log.Error(
"Workaround for ERROR_UNSUPPORTED_OS see http://bit.ly/fw7qzo FAILED");
Environment.Exit(-1);
}
Log.Info("Using temporary workaround for ERROR_UNSUPPORTED_OS see http://bit.ly/fw7qzo");
Thread.Sleep(10000);
}
}
public WorkerRoleIO()
{
this._LocalResource = RoleEnvironment.GetLocalResource("Regisfra.LocalStorage");
}
private CloudDriveSettings()
{
BlobContainerName = RoleEnvironment.GetConfigurationSettingValue(CloudDriveBlobContainerConfigOption);
BlobName = RoleEnvironment.GetConfigurationSettingValue(CloudDriveBlobNameConfigOption);
BlobSize = ushort.Parse(RoleEnvironment.GetConfigurationSettingValue(CloudDriveBlobSizeConfigOption));
ConnectionString = RoleEnvironment.GetConfigurationSettingValue(CloudDriveConnectionStringConfigOption);
InitCache = bool.Parse(RoleEnvironment.GetConfigurationSettingValue(CloudDriveInitCacheConfigOption));
CacheLocalResource = null;
if (InitCache)
{
var cacheResourceName = RoleEnvironment.GetConfigurationSettingValue(CloudDriveCacheLocalResourceNameConfigOption);
CacheLocalResource = RoleEnvironment.GetLocalResource(cacheResourceName);
}
}
private void OnInitialize()
{
var selfInstance = InstanceEnumerator.EnumerateInstances().First(i => i.IsSelf);
cloudStorageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue(ConfigurationSettingsKeys.StorageConnectionString));
log.Info("Storage account selected: {0}",cloudStorageAccount.BlobEndpoint);
cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
log.Info("Storage client created");
var containerName = ConfigurationProvider.GetSetting(ConfigurationSettingsKeys.StorageContainerName,"ravendb");
// In order to force a connection we just enumerate all available containers:
var availableContainers = cloudBlobClient.ListContainers().ToArray();
foreach (var container in availableContainers)
{
log.Info("Available container: {0}",container.Name);
}
if (!availableContainers.Any(c => c.Name.Equals(containerName)))
{
log.Info("Container {0} does not exist, creating",containerName);
// Container does not exist:
cloudBlobClient.GetContainerReference(containerName).Create();
}
cloudBlobContainer = cloudBlobClient.GetContainerReference(containerName);
log.Info("Container {0} selected",cloudBlobContainer.Name);
localCache = RoleEnvironment.GetLocalResource(ConfigurationSettingsKeys.StorageCacheResource);
log.Info("Cache resource retrieved: {0}, path: {1}",localCache.Name,localCache.RootPath);
CloudDrive.InitializeCache(localCache.RootPath, localCache.MaximumSizeInMegabytes);
log.Info("Cache initialized: {0} mb",localCache.MaximumSizeInMegabytes);
var driveName = string.Format("{0}{1}.vhd", selfInstance.InstanceType, selfInstance.InstanceIndex).ToLowerInvariant();
log.Info("Virtual drive name: {0}",driveName);
var pageBlob = cloudBlobContainer.GetPageBlobReference(driveName);
log.Info("Virtual drive blob: {0}",pageBlob.Uri);
cloudDrive = cloudStorageAccount.CreateCloudDrive(pageBlob.Uri.ToString());
log.Info("Virtual drive created: {0}",cloudDrive.Uri);
var storageSize = ConfigurationProvider.GetSetting(ConfigurationSettingsKeys.StorageSize, 50000);
log.Info("Storage size: {0} mb",storageSize);
cloudDrive.CreateIfNotExist(storageSize);
log.Info("Virtual drive initialized: {0}",cloudDrive.Uri);
var mountedDirectoryPath = cloudDrive.Mount(storageSize, DriveMountOptions.None);
log.Info("Virtual drive mounted at: {0}",mountedDirectoryPath);
mountedDirectory = new DirectoryInfo(mountedDirectoryPath);
log.Info("Ensuring drive is available: {0}",mountedDirectoryPath);
UpdateTestFile();
log.Info("Storage initialization succeeded");
}
private void InitializeLocalCache()
{
_localCache = RoleEnvironment.GetLocalResource("RavenCache");
CloudDrive.InitializeCache(_localCache.RootPath.TrimEnd('\\'), _localCache.MaximumSizeInMegabytes);
}
// Use this for initialization
protected virtual void Start()
{
dependencyLines = new Dictionary<GameObject, VectorLine>();
defaultColor = renderer.material.color;
if(transform.parent != null)
{
planet = transform.parent.GetComponent<Planet>();
if(planet != null)
{
//if(!CheckRequirements(planet))
// Destroy(gameObject);
}
}
//maintain global scale
if(transform.parent != null)
{
Vector3 correctedScale = new Vector3(transform.localScale.x / transform.parent.localScale.x,
transform.localScale.y / transform.parent.localScale.y,
transform.localScale.z / transform.parent.localScale.z);
transform.localScale = correctedScale;
}
//Initialize
input = new LocalResource[0];
output = new LocalResource();
output.type = LocalResourceType.Power;
output.quantity = 5;
outputRange = DEFAULT_OUTPUT_RANGE;
}
public HostedAzureLocalResource(LocalResource localResource)
{
_localResource = localResource;
}
