public void TestServerDirectoryFileDump()
{
string pathQuery = Guid.NewGuid().ToString();
string tempDir = Path.Combine(Path.GetTempPath(), pathQuery);
Directory.CreateDirectory(tempDir);
string loremIpsum = "Lorem ipsum dolor sit amet";
StringBuilder sb = new StringBuilder(loremIpsum.Length * 1024);
for (int i = 0; i < 102400; i++)
{
sb.Append(loremIpsum);
}
File.WriteAllText(Path.Combine(tempDir, "file.txt"), sb.ToString());
MockDeaClient.SetResponse(tempDir);
MockDeaClient client = new MockDeaClient();
CloudFoundry.WinDEA.DirectoryServer.DirectoryServer server = new CloudFoundry.WinDEA.DirectoryServer.DirectoryServer();
server.Start("127.0.0.1", DirectoryConfiguration.ReadConfig(), client);
string output = DownloadString(string.Format("http://127.0.0.1:{0}/{1}", DirectoryConfiguration.ReadConfig().DirectoryServer.V2Port, "file.txt"));
server.Stop();
Assert.IsFalse(string.IsNullOrWhiteSpace(output));
Assert.IsTrue(output == sb.ToString());
}
private void InitializeWadConfiguration()
{
String wadConnectionString = "Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString";
String customContainerName = "wad-custom-container";
DiagnosticMonitorConfiguration dmc = DiagnosticMonitor.GetDefaultInitialConfiguration();
LocalResource localResource = RoleEnvironment.GetLocalResource(localResourceName);
String logPath = Path.Combine(localResource.RootPath, "Logs");
DirectoryConfiguration directoryConfiguration = new DirectoryConfiguration()
{
Container = customContainerName,
DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes,
Path = logPath
};
dmc.Directories.DataSources.Add(directoryConfiguration);
dmc.Directories.ScheduledTransferPeriod = TimeSpan.FromHours(1);
dmc.Logs.BufferQuotaInMB = 100;
dmc.Logs.ScheduledTransferPeriod = TimeSpan.FromHours(1);
dmc.Logs.ScheduledTransferLogLevelFilter = LogLevel.Verbose;
CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue(wadConnectionString));
DiagnosticMonitor.Start(wadConnectionString, dmc);
}
public override bool OnStart()
{
// Configure diagnostics
// From: http://www.packtpub.com/article/digging-into-windows-azure-diagnostics-tutotial (sic),
// need to add a diagnostics trace listener; (web.config is just for website tracing)
Trace.Listeners.Add(new DiagnosticMonitorTraceListener());
Trace.AutoFlush = true;
Trace.TraceInformation("Error");
// Set up to transfer to blob storage
DiagnosticMonitorConfiguration diagnosticConfiguration = DiagnosticMonitor.GetDefaultInitialConfiguration();
diagnosticConfiguration.Logs.ScheduledTransferPeriod = TimeSpan.FromMinutes(10);
// don't need; if not specificed, will use overall quota (which is 4 GB) diagnosticConfiguration.Logs.BufferQuotaInMB = 10;
// Add the startup task logs directory, so the startup logs will get transferred to storage
var dirConfig = new DirectoryConfiguration();
dirConfig.Container = "wad-startuplogs-container";
dirConfig.Path = Path.Combine((new DirectoryInfo(".")).FullName, "startuplogs");
Trace.TraceInformation("not sure why files not getting picked up: dirConfig = " + dirConfig.Path);
diagnosticConfiguration.Directories.DataSources.Add(dirConfig);
diagnosticConfiguration.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(10);
DiagnosticMonitor.Start("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString", diagnosticConfiguration);
// For information on handling configuration changes
// see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357.
return(base.OnStart());
}
public override bool OnStart()
{
// Configure diagnostics
// From: http://www.packtpub.com/article/digging-into-windows-azure-diagnostics-tutotial (sic),
// need to add a diagnostics trace listener; (web.config is just for website tracing)
Trace.Listeners.Add(new DiagnosticMonitorTraceListener());
Trace.AutoFlush = true;
Trace.TraceInformation("Error");
// Set up to transfer to blob storage
DiagnosticMonitorConfiguration diagnosticConfiguration = DiagnosticMonitor.GetDefaultInitialConfiguration();
diagnosticConfiguration.Logs.ScheduledTransferPeriod = TimeSpan.FromMinutes(10);
// don't need; if not specificed, will use overall quota (which is 4 GB) diagnosticConfiguration.Logs.BufferQuotaInMB = 10;
// Add the startup task logs directory, so the startup logs will get transferred to storage
var dirConfig = new DirectoryConfiguration();
dirConfig.Container = "wad-startuplogs-container";
dirConfig.Path = Path.Combine((new DirectoryInfo(".")).FullName, "startuplogs");
Trace.TraceInformation("not sure why files not getting picked up: dirConfig = " + dirConfig.Path);
diagnosticConfiguration.Directories.DataSources.Add(dirConfig);
diagnosticConfiguration.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(10);
DiagnosticMonitor.Start("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString", diagnosticConfiguration);
// For information on handling configuration changes
// see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357.
return base.OnStart();
}
/// <summary>
/// Removes a directory from the cleanup configuration
/// </summary>
/// <param name="path">File path of the directory to remove</param>
public void RemoveDirectory(string path)
{
if (!string.IsNullOrWhiteSpace(path))
{
if (_config != null)
{
_logger.ShowInformation(string.Format("Removing directory: '{0}'", path));
lock (_config)
{
DirectoryConfiguration dir = null;
foreach (DirectoryConfiguration d in _config.Directories)
{
if (string.Compare(path.TrimEnd('\\'), d.Path.TrimEnd('\\'), true) == 0)
{
dir = d;
break;
}
}
if (dir != null)
{
_config.Directories.Remove(dir);
SaveConfiguration();
}
}
}
}
}
public AuthConfigurations(IConfiguration configuration)
{
var sections = configuration.GetSection("Authorization").GetChildren();
foreach (IConfigurationSection section in sections)
{
if (section["Type"] == "ActiveDirectory")
{
DirectoryConfiguration conf = section.Get <DirectoryConfiguration>();
_configs.Add(conf);
this.ConfigurationNames.Add(conf.Name);
}
else if (section["Type"] == "LocalServer")
{
LocalAuthConfiguration conf = section.Get <LocalAuthConfiguration>();
_configs.Add(conf);
this.ConfigurationNames.Add(conf.Name);
}
else if (section["Type"] == "LDAP")
{
LdapConfiguration conf = section.Get <LdapConfiguration>();
_configs.Add(conf);
this.ConfigurationNames.Add(conf.Name);
}
}
}
public async Task ShouldEmitEventFromCluster()
{
IActor actorFactory() => new HelloActorHandleEvent();
var clusterConfiguration1 = new ClusterConfiguration("TestCluster1", "http://127.0.0.1:8080", actorFactory);
var clusterConfiguration2 = new ClusterConfiguration("TestCluster2", "http://127.0.0.1:8181", actorFactory);
var directoryConfiguration = new DirectoryConfiguration(TimeSpan.FromSeconds(30));
var cluster1 = Cluster.Create <TestClusterRegistry>(clusterConfiguration1, directoryConfiguration);
var process1 = cluster1.Spawn(actorFactory, "http://127.0.0.1:8282");
var cluster2 = Cluster.Create <TestClusterRegistry>(clusterConfiguration2, directoryConfiguration);
var process2 = cluster1.Spawn(actorFactory, "http://127.0.0.1:8383");
cluster1.Emit("http://127.0.0.1:8383", new Hello(process1.Configuration.Id.Value));
await Task.Delay(10);
var actor = process2.Actor as HelloActorHandleEvent;
Assert.IsNotNull(actor);
Assert.AreEqual(1, actor.Received.Count);
cluster1.Dispose();
cluster2.Dispose();
}
private void AreEqual(DirectoryConfiguration exp, DirectoryConfiguration act)
{
if (exp != null)
{
Assert.Equal(exp.ContainerName, act.ContainerName);
AreEqual(exp.Path, act.Path);
}
}
private static DirectoryConfiguration CreateDirectoryConfiguration(LocalResource localResource)
{
DirectoryConfiguration dirConfig = new DirectoryConfiguration();
dirConfig.Container = "diagnostics-customlogfiles-container";
dirConfig.DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes;
dirConfig.Path = localResource.RootPath;
return dirConfig;
}
public static DirectoryConfiguration GetLogDirectory()
{
DirectoryConfiguration directory = new DirectoryConfiguration();
directory.Container = "wad-tracefiles";
directory.DirectoryQuotaInMB = 10;
directory.Path = RoleEnvironment.GetLocalResource("CoreService.svclog").RootPath;
return directory;
}
public static DirectoryConfiguration GetLogDirectory()
{
DirectoryConfiguration directory = new DirectoryConfiguration();
directory.Container = "wad-tracefiles";
directory.DirectoryQuotaInMB = 10;
directory.Path = RoleEnvironment.GetLocalResource("WCFServiceSurferlite.svclog").RootPath;
return directory;
}
public void SetDirectoryConfiguration()
{
if (DirectoryConfiguration == null)
{
DirectoryConfiguration = new DirectoryConfiguration(DirectoriesManager);
DirectoryConfiguration.StartWatchingAsync(DirectoryConfigurationCallback, null);
}
}
private static LocalResource CreateLocalResource(DiagnosticMonitorConfiguration diagMonitorConfig)
{
LocalResource localResource = RoleEnvironment.GetLocalResource("CustomLogFiles");
DirectoryConfiguration dirConfig = CreateDirectoryConfiguration(localResource);
diagMonitorConfig.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(1.0);
diagMonitorConfig.Directories.DataSources.Add(dirConfig);
return(localResource);
}
private static DirectoryConfiguration CreateDirectoryConfiguration(LocalResource localResource)
{
DirectoryConfiguration dirConfig = new DirectoryConfiguration();
dirConfig.Container = "diagnostics-customlogfiles-container";
dirConfig.DirectoryQuotaInMB = localResource.MaximumSizeInMegabytes;
dirConfig.Path = localResource.RootPath;
return(dirConfig);
}
public SingleThreadedRecoveryTest()
{
// Create FASTER index
var log = FASTERFactory.CreateLogDevice(DirectoryConfiguration.GetHybridLogFileName());
fht = FASTERFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFaster>
(keySpace, log);
}
public DirectoryPopulationService(ApplicationDbContext dbContext, HttpClient httpClient, IOptions <DirectoryConfiguration> config)
{
this.dbContext = dbContext;
this.httpClient = httpClient;
this.config = config.Value;
// TODO: get key from settings
this.peopleLookupKey = this.config.ApiKey;
}
/// <summary>
/// ログ出力初期化
/// </summary>
private void InitLog()
{
var transferPeriod = TimeSpan.FromMinutes(10);
var bufferQuotaInMB = 100;
DiagnosticMonitorConfiguration dmc = DiagnosticMonitor.GetDefaultInitialConfiguration();
dmc.Logs.ScheduledTransferLogLevelFilter = LogLevel.Information;
dmc.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(10);
LocalResource localResource = RoleEnvironment.GetLocalResource(WEB_ROLE_LOCAL_STRAGE_NAME);
DirectoryConfiguration dc = new DirectoryConfiguration()
{
Container = LOG_CONTAINER_NAME,
DirectoryQuotaInMB = bufferQuotaInMB,
Path = localResource.RootPath
};
dmc.Directories.DataSources.Add(dc);
// Add Windows Azure Trace Listener
System.Diagnostics.Trace.Listeners.Add(new Microsoft.WindowsAzure.Diagnostics.DiagnosticMonitorTraceListener());
// Enable Collection of Crash Dumps
CrashDumps.EnableCollection(true);
// File-based logs
dmc.DiagnosticInfrastructureLogs.ScheduledTransferPeriod = transferPeriod;
dmc.DiagnosticInfrastructureLogs.BufferQuotaInMB = bufferQuotaInMB;
dmc.DiagnosticInfrastructureLogs.ScheduledTransferLogLevelFilter = LogLevel.Warning;
// Windows Event logs
dmc.WindowsEventLog.DataSources.Add("Application!*");
dmc.WindowsEventLog.DataSources.Add("System!*");
dmc.WindowsEventLog.ScheduledTransferPeriod = transferPeriod;
dmc.WindowsEventLog.ScheduledTransferLogLevelFilter = LogLevel.Information;
dmc.WindowsEventLog.BufferQuotaInMB = bufferQuotaInMB;
// Performance Counters
var counters = new List<string> {
@"\Processor(_Total)\% Processor Time",
@"\Memory\Available MBytes",
@"\ASP.NET Applications(__Total__)\Requests Total",
@"\ASP.NET Applications(__Total__)\Requests/Sec",
@"\ASP.NET\Requests Queued",
};
counters.ForEach(
counter =>
{
dmc.PerformanceCounters.DataSources.Add(
new PerformanceCounterConfiguration { CounterSpecifier = counter, SampleRate = TimeSpan.FromSeconds(60) });
});
dmc.PerformanceCounters.ScheduledTransferPeriod = transferPeriod;
dmc.PerformanceCounters.BufferQuotaInMB = bufferQuotaInMB;
DiagnosticMonitor.Start("Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString", dmc);
}
public static DirectoryConfiguration GetLogDirectory()
{
DirectoryConfiguration directory = new DirectoryConfiguration();
directory.Container = "wad-tracefiles";
directory.DirectoryQuotaInMB = 10;
directory.Path = RoleEnvironment.GetLocalResource("ContactData.svclog").RootPath;
return(directory);
}
/// <summary>
/// Gets the log directory configuration.
/// </summary>
/// <returns>Returns the log directory configuration.</returns>
public static DirectoryConfiguration GetLogDirectory()
{
DirectoryConfiguration directory = new DirectoryConfiguration();
directory.Container = "wad-tracefiles";
directory.DirectoryQuotaInMB = 10;
directory.Path = RoleEnvironment.GetLocalResource("Microsoft.Whitespace.Sync.Database.Service.svclog").RootPath;
return(directory);
}
public static DirectoryConfiguration GetLogDirectory()
{
var directory = new DirectoryConfiguration
{
Container = "wad-tracefiles",
DirectoryQuotaInMB = 10,
Path = RoleEnvironment.GetLocalResource("Cornball.Service.svclog").RootPath
};
return directory;
}
public void Icon_SetGet_ReturnsSetValue()
{
DirectoryConfiguration directoryConfiguration = CreateDirectoryConfiguration();
var expectedMockIconConfiguration = Substitute.For <IExplorerTreeItemIconConfiguration>();
directoryConfiguration.Icon = expectedMockIconConfiguration;
Assert.AreEqual(expectedMockIconConfiguration, directoryConfiguration.Icon);
}
public static DirectoryConfiguration GetLogDirectory()
{
DirectoryConfiguration directory = new DirectoryConfiguration();
directory.Container = "wad-tracefiles";
directory.DirectoryQuotaInMB = 10;
if (RoleEnvironment.IsAvailable)
directory.Path = RoleEnvironment.GetLocalResource("ActivityServiceWebRole.svclog").RootPath;
else
directory.Path = "c:\\";
return directory;
}
public void WatchDirectory(DirectoryMetadataCategory directoryMetaDataCategory)
{
foreach (DirectoryMetadata d in DirectoryConfiguration.GetList())
{
if (d.Category == directoryMetaDataCategory)
{
WatchedDirectory = new ContactDirectory(DirectoriesManager, d);
WatchedDirectory.StartWatching();
}
}
}
public List <DirectoryMetadataCategory> GetDirectoryCategories()
{
List <DirectoryMetadataCategory> tempList = new List <DirectoryMetadataCategory>();
foreach (DirectoryMetadata d in DirectoryConfiguration.GetList())
{
tempList.Add(d.Category);
}
List <DirectoryMetadataCategory> distinctOnly = new List <DirectoryMetadataCategory>(new HashSet <DirectoryMetadataCategory>(tempList));
return(distinctOnly);
}
public void ShouldCreateCluster()
{
IActor actorFactory() => new HelloActorHandleEvent();
var clusterConfiguration = new ClusterConfiguration("TestCluster", "http://127.0.0.1:8080", actorFactory);
var directoryConfiguration = new DirectoryConfiguration(TimeSpan.FromSeconds(30));
var cluster = Cluster.Create <TestClusterRegistry>(clusterConfiguration, directoryConfiguration);
Assert.IsNotNull(cluster);
cluster.Dispose();
}
public List <DirectoryMetadata> GetDirectoriesFromCategory(DirectoryMetadataCategory Category)
{
List <DirectoryMetadata> tempList = new List <DirectoryMetadata>();
foreach (DirectoryMetadata d in DirectoryConfiguration.GetList())
{
if (d.Category == Category)
{
tempList.Add(d);
}
}
return(tempList);
}
public static List <DirectoryConfiguration> GetListOfRefDirectoryFromConfigFile(String path)
{
XDocument doc = XDocument.Load(path);
List <DirectoryConfiguration> dirConfigurations = new List <DirectoryConfiguration>();
foreach (XElement reference in doc.Root.Elements("reference"))
{
var configuration = new DirectoryConfiguration();
configuration.NugetRefDirectory = reference.Element("NugetReference").Value;
configuration.ProjectRefeDirectory = reference.Element("ProjectReference").Value;
dirConfigurations.Add(configuration);
}
return(dirConfigurations);
}
public ConcurrentTest(int threadCount)
{
this.threadCount = threadCount;
// Create FASTER index
var log = FASTERFactory.CreateLogDevice(DirectoryConfiguration.GetHybridLogFileName());
fht = FASTERFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFaster>
(keySpace, log);
numActiveThreads = 0;
inputArrays = new BlockingCollection <Input[]>();
threadNumOps = new long[threadCount];
Prepare();
}
public ConcurrentRecoveryTest(int threadCount)
{
this.threadCount = threadCount;
tokens = new List <Guid>();
var log = FasterFactory.CreateLogDevice(DirectoryConfiguration.GetHybridLogFileName());
// Create FASTER index
fht = FasterFactory.Create
<AdId, NumClicks, Input, Output, Empty, Functions, ICustomFasterKv>
(keySpace, log);
numActiveThreads = 0;
inputArrays = new BlockingCollection <Input[]>();
Prepare();
}
public void TestServerDirectoryDirList()
{
string pathQuery = Guid.NewGuid().ToString();
string tempDir = Path.Combine(Path.GetTempPath(), pathQuery);
Directory.CreateDirectory(tempDir);
Directory.CreateDirectory(Path.Combine(tempDir, "dir1"));
Directory.CreateDirectory(Path.Combine(tempDir, "dir2"));
string loremIpsum = "Lorem ipsum dolor sit amet";
File.WriteAllText(Path.Combine(tempDir, "file1.txt"), loremIpsum);
StringBuilder sb = new StringBuilder(loremIpsum.Length * 1024);
for (int i = 0; i < 1024; i++)
{
sb.Append(loremIpsum);
}
File.WriteAllText(Path.Combine(tempDir, "file2.txt"), sb.ToString());
MockDeaClient.SetResponse(Path.GetTempPath());
MockDeaClient client = new MockDeaClient();
Uhuru.CloudFoundry.DEA.DirectoryServer.DirectoryServer server = new DEA.DirectoryServer.DirectoryServer();
server.Start("127.0.0.1", DirectoryConfiguration.ReadConfig(), client);
string output = DownloadString(string.Format("http://127.0.0.1:{0}/{1}", DirectoryConfiguration.ReadConfig().DirectoryServer.V2Port, pathQuery));
server.Stop();
Assert.IsFalse(string.IsNullOrWhiteSpace(output));
string expectedOutput = string.Format(@"dir1/ -
dir2/ -
file1.txt {0}B
file2.txt {1}K
", loremIpsum.Length, ((1024.0 * loremIpsum.Length) / 1024).ToString("0.00"));
Assert.IsTrue(output == expectedOutput);
}
/// <summary>
/// Adds a directory to the cleanup congfiguration
/// </summary>
/// <param name="path">File path for the directory</param>
/// <param name="isRecursive">True for recursive cleanup. False for top level only</param>
/// <param name="action"><see cref="CleanupAction"/> type</param>
public void AddDirectory(string path, bool isRecursive, CleanupAction action)
{
if (!string.IsNullOrWhiteSpace(path))
{
if (_config != null)
{
_logger.ShowInformation(string.Format("Adding directory: '{0}' Recursive: {1} Action: {2}", path, isRecursive, action));
lock (_config)
{
DirectoryConfiguration dir = null;
foreach (DirectoryConfiguration d in _config.Directories)
{
if (string.Compare(path.TrimEnd('\\'), d.Path.TrimEnd('\\'), true) == 0)
{
dir = d;
break;
}
}
if (dir == null)
{
dir = new DirectoryConfiguration();
}
else
{
_config.Directories.Remove(dir);
}
dir.Path = path;
dir.CleanupAction = action;
dir.Recursive = isRecursive;
_config.Directories.Add(dir);
SaveConfiguration();
}
}
}
}
/// <summary>Windows Azure診断（Windows Azure Diagnostics）関連の設定</summary>
private void ConfigureDiagnostics()
{
// Windows Azure診断（Windows Azure Diagnostics）情報出力用のストレージ アカウント取得
string wadConnectionString = "Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString";
//// ストレージ アカウントの初期化
//CloudStorageAccount cloudStorageAccount =
// CloudStorageAccount.Parse(
// RoleEnvironment.GetConfigurationSettingValue(wadConnectionString));
//// RoleInstance診断管理の初期化（ロールに対する診断の有効化）
//RoleInstanceDiagnosticManager roleInstanceDiagnosticManager =
// cloudStorageAccount.CreateRoleInstanceDiagnosticManager(
// RoleEnvironment.DeploymentId,
// RoleEnvironment.CurrentRoleInstance.Role.Name,
// RoleEnvironment.CurrentRoleInstance.Id);
var storageConnectionString = RoleEnvironment.GetConfigurationSettingValue(
wadConnectionString);
var deploymentDiagnosticManager = new DeploymentDiagnosticManager(
storageConnectionString,
RoleEnvironment.DeploymentId);
RoleInstanceDiagnosticManager roleInstanceDiagnosticManager = deploymentDiagnosticManager.GetRoleInstanceDiagnosticManager(
RoleEnvironment.CurrentRoleInstance.Role.Name,
RoleEnvironment.CurrentRoleInstance.Id);
// RoleInstance診断管理からコンフィグを取得する。
// ・基本的にはデフォルト設定を利用する。
// ・RoleEnvironment.Changedイベント ハンドラを仕掛ける場合はカレント設定を利用する。
DiagnosticMonitorConfiguration config =
DiagnosticMonitor.GetDefaultInitialConfiguration();
// roleInstanceDiagnosticManager.GetCurrentConfiguration();
#region Windows Azure診断（Windows Azure Diagnostics）APIを使用した設定開始
#region インフラストラクチャ ログ（診断モニタ自体のログ）
// 転送レベルおよび転送間隔を設定
config.DiagnosticInfrastructureLogs.ScheduledTransferLogLevelFilter = LogLevel.Undefined; // 要検討
config.DiagnosticInfrastructureLogs.ScheduledTransferPeriod = TimeSpan.FromSeconds(15); // 要検討
#endregion
// 出力先のTableストレージ名：WADDiagnosticInfrastructureLogsTable
#region イベント ログの設定
// 取得するイベント ソースを設定
config.WindowsEventLog.DataSources.Add("Application!*");
config.WindowsEventLog.DataSources.Add("System!*");
// 転送レベルおよび転送間隔を設定
config.WindowsEventLog.ScheduledTransferLogLevelFilter = LogLevel.Undefined; // 要検討
config.WindowsEventLog.ScheduledTransferPeriod = TimeSpan.FromMinutes(15); // 要検討
#endregion
// 出力先のTableストレージ名：WADWindowsEventLogsTable
#region パフォーマンス カウンタの転送設定
// カウンタ、サンプリング レートの指定
config.PerformanceCounters.DataSources.Add(new PerformanceCounterConfiguration()
{
CounterSpecifier = @"\Processor(_Total)\% Processor Time",
SampleRate = TimeSpan.FromSeconds(10)
});
config.PerformanceCounters.DataSources.Add(new PerformanceCounterConfiguration()
{
CounterSpecifier = @"\Memory\Available Bytes",
SampleRate = TimeSpan.FromSeconds(10)
});
// 転送間隔を設定
config.PerformanceCounters.ScheduledTransferPeriod = TimeSpan.FromMinutes(15); // 要検討
#endregion
// 出力先のTableストレージ名：WADPerformanceCountersTable
#region クラッシュ ダンプ転送の有効化
CrashDumps.EnableCollection(true);
#endregion
// 出力先のBlobストレージ コンテナ名：wad-crash-dumps
#region IISログ、FREBログの転送設定(※ web.config への設定も必要)
// IISログはデフォルトで取得が有効となっているため、Blobへの転送を指定するのみで収集が可能となる。
// FREB（Failed Request Trace log）ログについてはweb.config への設定も必要
#endregion
// 出力先のBlobストレージ コンテナ名：wad-iis-logfiles、wad-iis-failedreqlogfiles
#region トレース ログの設定(※ *.config への設定も必要)
// 転送レベルおよび転送間隔を設定
config.Logs.ScheduledTransferLogLevelFilter = LogLevel.Undefined; // 要検討
config.Logs.ScheduledTransferPeriod = TimeSpan.FromMinutes(1); // 要検討
#endregion
// 出力先のTableストレージ名：WADLogsTable
#region カスタム ログ（ローカル ストレージへの出力の場合）
// 出力先ディレクトリ設定
DirectoryConfiguration dirConfig1 = new DirectoryConfiguration()
{
// 出力先Blobコンテナの指定
Container = "my-custom-logfiles1",
// クォーターの設定（実際に使う量）
DirectoryQuotaInMB = 100,
};
// ローカル ストレージのパスを指定
//（ローカル ストレージの設定自体は、Visual Studioを使用し*.csdefに設定可能）
LocalResource ls = RoleEnvironment.GetLocalResource("LogStorage");
dirConfig1.Path = ls.RootPath;
// log4netに環境変数経由でパス情報（RootPath）を
// 渡そうとしたがエミュレータ上でうまく動作せず断念。
// ローカル ストレージを転送元コレクションに追加
config.Directories.DataSources.Add(dirConfig1);
// なお、ローカル ストレージのパスは、LocalResource.RootPathにて取得が可能である。
#endregion
// 出力先のBlobストレージ コンテナ名：my-custom-logfiles1
#region カスタムログ（任意の出力先の場合）
// 出力先ディレクトリ設定
DirectoryConfiguration dirConfig2 = new DirectoryConfiguration()
{
// 出力先Blobコンテナの指定
Container = "my-custom-logfiles2",
// クォーターの設定（実際に使う量）
DirectoryQuotaInMB = 100,
};
// 任意のディレクトリを指定
string path = "c:\\logs";
dirConfig2.Path = path;
// ディレクトリを転送元コレクションに追加
config.Directories.DataSources.Add(dirConfig2);
// ディレクトリ セキュリティを取得
DirectorySecurity ds = Directory.GetAccessControl(path);
// Everyone FullControlのアクセス ルールの生成
FileSystemAccessRule AccessRule = new FileSystemAccessRule(
"Everyone",
FileSystemRights.FullControl,
InheritanceFlags.ObjectInherit,
PropagationFlags.None,
AccessControlType.Allow);
// ディレクトリ キュリティにアクセス ルールを追加
ds.AddAccessRule(AccessRule);
// ディレクトリにディレクトリ セキュリティを反映
// ★ <Runtime executionContext="elevated"/>を「*.csdef」に記述。
Directory.SetAccessControl(path, ds);
#endregion
// 出力先のBlobストレージ コンテナ名：my-custom-logfiles2
// IISログ、カスタム ログ、クラッシュ ダンプなどで
// 使用するディレクトリ バッファからBlobストレージへの転送間隔の指定
config.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(15);
#endregion
// RoleInstance診断管理にコンフィグを設定
roleInstanceDiagnosticManager.SetCurrentConfiguration(config);
// 診断の開始（エミュレータでは不要だが、実機では必要
DiagnosticMonitor.Start(wadConnectionString, config);
// ローカル ストレージへのログ出力テスト
path = Path.Combine(ls.RootPath,
string.Format("test_{0}.txt", DateTime.Now.ToString("yyyyMMdd")));
// StreamWriterを開き、ログを出力
using (StreamWriter sw = new StreamWriter(path, true))
{
sw.WriteLine("{0} : {1}", DateTime.UtcNow, "ローカル ストレージへのログ出力テスト");
sw.Close();
}
}
/// <summary>Windows Azure診断（Windows Azure Diagnostics）関連の設定</summary>
private void ConfigureDiagnostics()
{
// Windows Azure診断（Windows Azure Diagnostics）情報出力用のストレージ アカウント取得
string wadConnectionString = "Microsoft.WindowsAzure.Plugins.Diagnostics.ConnectionString";
// ストレージ アカウントの初期化
CloudStorageAccount cloudStorageAccount =
CloudStorageAccount.Parse(
RoleEnvironment.GetConfigurationSettingValue(wadConnectionString));
// RoleInstance診断管理の初期化（ロールに対する診断の有効化）
RoleInstanceDiagnosticManager roleInstanceDiagnosticManager =
cloudStorageAccount.CreateRoleInstanceDiagnosticManager(
RoleEnvironment.DeploymentId,
RoleEnvironment.CurrentRoleInstance.Role.Name,
RoleEnvironment.CurrentRoleInstance.Id);
// RoleInstance診断管理からコンフィグを取得する。
// ・基本的にはデフォルト設定を利用する。
// ・RoleEnvironment.Changedイベント ハンドラを仕掛ける場合はカレント設定を利用する。
DiagnosticMonitorConfiguration config =
DiagnosticMonitor.GetDefaultInitialConfiguration();
// roleInstanceDiagnosticManager.GetCurrentConfiguration();
#region Windows Azure診断（Windows Azure Diagnostics）APIを使用した設定開始
#region インフラストラクチャ ログ（診断モニタ自体のログ）
// 転送レベルおよび転送間隔を設定
config.DiagnosticInfrastructureLogs.ScheduledTransferLogLevelFilter = LogLevel.Undefined; // 要検討
config.DiagnosticInfrastructureLogs.ScheduledTransferPeriod = TimeSpan.FromSeconds(15); // 要検討
#endregion
// 出力先のTableストレージ名：WADDiagnosticInfrastructureLogsTable
#region イベント ログの設定
// 取得するイベント ソースを設定
config.WindowsEventLog.DataSources.Add("Application!*");
config.WindowsEventLog.DataSources.Add("System!*");
// 転送レベルおよび転送間隔を設定
config.WindowsEventLog.ScheduledTransferLogLevelFilter = LogLevel.Undefined; // 要検討
config.WindowsEventLog.ScheduledTransferPeriod = TimeSpan.FromMinutes(15); // 要検討
#endregion
// 出力先のTableストレージ名：WADWindowsEventLogsTable
#region パフォーマンス カウンタの転送設定
// カウンタ、サンプリング レートの指定
config.PerformanceCounters.DataSources.Add(new PerformanceCounterConfiguration()
{
CounterSpecifier = @"\Processor(_Total)\% Processor Time",
SampleRate = TimeSpan.FromSeconds(10)
});
config.PerformanceCounters.DataSources.Add(new PerformanceCounterConfiguration()
{
CounterSpecifier = @"\Memory\Available Bytes",
SampleRate = TimeSpan.FromSeconds(10)
});
// 転送間隔を設定
config.PerformanceCounters.ScheduledTransferPeriod = TimeSpan.FromMinutes(15); // 要検討
#endregion
// 出力先のTableストレージ名：WADPerformanceCountersTable
#region クラッシュ ダンプ転送の有効化
CrashDumps.EnableCollection(true);
#endregion
// 出力先のBlobストレージ コンテナ名：wad-crash-dumps
#region IISログ、FREBログの転送設定(※ web.config への設定も必要)
// IISログはデフォルトで取得が有効となっているため、Blobへの転送を指定するのみで収集が可能となる。
// FREB（Failed Request Trace log）ログについてはweb.config への設定も必要
#endregion
// 出力先のBlobストレージ コンテナ名：wad-iis-logfiles、wad-iis-failedreqlogfiles
#region トレース ログの設定(※ *.config への設定も必要)
// 転送レベルおよび転送間隔を設定
config.Logs.ScheduledTransferLogLevelFilter = LogLevel.Undefined; // 要検討
config.Logs.ScheduledTransferPeriod = TimeSpan.FromMinutes(1); // 要検討
#endregion
// 出力先のTableストレージ名：WADLogsTable
#region カスタム ログ（ローカル ストレージへの出力の場合）
// 出力先ディレクトリ設定
DirectoryConfiguration dirConfig1 = new DirectoryConfiguration()
{
// 出力先Blobコンテナの指定
Container = "my-custom-logfiles1",
// クォーターの設定（実際に使う量）
DirectoryQuotaInMB = 100,
};
// ローカル ストレージのパスを指定
//（ローカル ストレージの設定自体は、Visual Studioを使用し*.csdefに設定可能）
LocalResource ls = RoleEnvironment.GetLocalResource("LogStorage");
dirConfig1.Path = ls.RootPath;
// log4netに環境変数経由でパス情報（RootPath）を
// 渡そうとしたがエミュレータ上でうまく動作せず断念。
// ローカル ストレージを転送元コレクションに追加
config.Directories.DataSources.Add(dirConfig1);
// なお、ローカル ストレージのパスは、LocalResource.RootPathにて取得が可能である。
#endregion
// 出力先のBlobストレージ コンテナ名：my-custom-logfiles1
#region カスタムログ（任意の出力先の場合）
// 出力先ディレクトリ設定
DirectoryConfiguration dirConfig2 = new DirectoryConfiguration()
{
// 出力先Blobコンテナの指定
Container = "my-custom-logfiles2",
// クォーターの設定（実際に使う量）
DirectoryQuotaInMB = 100,
};
// 任意のディレクトリを指定
string path = "c:\\logs";
dirConfig2.Path = path;
// ディレクトリを転送元コレクションに追加
config.Directories.DataSources.Add(dirConfig2);
// ディレクトリ セキュリティを取得
DirectorySecurity ds = Directory.GetAccessControl(path);
// Everyone FullControlのアクセス ルールの生成
FileSystemAccessRule AccessRule = new FileSystemAccessRule(
"Everyone",
FileSystemRights.FullControl,
InheritanceFlags.ObjectInherit,
PropagationFlags.None,
AccessControlType.Allow);
// ディレクトリ キュリティにアクセス ルールを追加
ds.AddAccessRule(AccessRule);
// ディレクトリにディレクトリ セキュリティを反映
// ★ <Runtime executionContext="elevated"/>を「*.csdef」に記述。
Directory.SetAccessControl(path, ds);
#endregion
// 出力先のBlobストレージ コンテナ名：my-custom-logfiles2
// IISログ、カスタム ログ、クラッシュ ダンプなどで
// 使用するディレクトリ バッファからBlobストレージへの転送間隔の指定
config.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(15);
#endregion
// RoleInstance診断管理にコンフィグを設定
roleInstanceDiagnosticManager.SetCurrentConfiguration(config);
// 診断の開始（エミュレータでは不要だが、実機では必要
DiagnosticMonitor.Start(wadConnectionString, config);
// ローカル ストレージへのログ出力テスト
path = Path.Combine(ls.RootPath,
string.Format("test_{0}.txt", DateTime.Now.ToString("yyyyMMdd")));
// StreamWriterを開き、ログを出力
using (StreamWriter sw = new StreamWriter(path, true))
{
sw.WriteLine("{0} : {1}", DateTime.UtcNow, "ローカル ストレージへのログ出力テスト");
sw.Close();
}
}
public void TestServerDirectoryFileTail()
{
string pathQuery = Guid.NewGuid().ToString();
string tempDir = Path.Combine(Path.GetTempPath(), pathQuery);
Directory.CreateDirectory(tempDir);
string loremIpsum = "Lorem ipsum dolor sit amet";
string filePath = Path.Combine(tempDir, "file.txt");
File.WriteAllText(filePath, loremIpsum);
MockDeaClient.SetResponse(tempDir);
MockDeaClient client = new MockDeaClient();
CloudFoundry.WinDEA.DirectoryServer.DirectoryServer server = new CloudFoundry.WinDEA.DirectoryServer.DirectoryServer();
DEAElement config = DirectoryConfiguration.ReadConfig();
config.DirectoryServer.StreamingTimeoutMS = 5000;
server.Start("127.0.0.1", config, client);
Random rnd = new Random();
string returnBytes = string.Empty;
string sentBytes = string.Empty;
int readCount = 0;
string appendChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
Thread newThread = new Thread(() =>
{
byte[] randomBytes = new byte[rnd.Next(100)];
rnd.NextBytes(randomBytes);
File.AppendAllText(filePath, ASCIIEncoding.ASCII.GetString(randomBytes));
HttpWebResponse response = null;
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(string.Format("http://127.0.0.1:{0}/{1}?tail", DirectoryConfiguration.ReadConfig().DirectoryServer.V2Port, "file.txt"));
response = (HttpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
byte[] responseBytes = new byte[200];
int read;
do
{
read = responseStream.Read(responseBytes, 0, responseBytes.Length);
returnBytes += ASCIIEncoding.ASCII.GetString(responseBytes, 0, read);
readCount++;
}while (read != 0);
});
Thread writerThread = new Thread(() =>
{
for (int i = 0; i < 10; i++)
{
Thread.Sleep(50);
string toWrite = string.Empty;
for (int j = 0; j < rnd.Next(100); j++)
{
toWrite += appendChars[rnd.Next(appendChars.Length)];
}
File.AppendAllText(filePath, toWrite);
sentBytes += toWrite;
}
});
newThread.Start();
Thread.Sleep(1000);
writerThread.Start();
writerThread.Join();
newThread.Join();
server.Stop();
Assert.AreEqual(sentBytes, returnBytes);
Assert.IsTrue(readCount > 1);
}
private void ConfigureProfileStorage()
{
LocalResource profileStorage;
try
{
profileStorage = RoleEnvironment.GetLocalResource("ProfileStorage");
}
catch (RoleEnvironmentException e)
{
throw new InvalidOperationException("Local storage \"ProfileStorage\" not found.", e);
}
var ridm = new RoleInstanceDiagnosticManager(
CloudConfigurationManager.GetSetting("AzurePerfTools.PowerShellWindowsService.ConnectionString"),
RoleEnvironment.DeploymentId,
RoleEnvironment.CurrentRoleInstance.Role.Name,
RoleEnvironment.CurrentRoleInstance.Id);
var dmc = ridm.GetCurrentConfiguration() ?? DiagnosticMonitor.GetDefaultInitialConfiguration();
DirectoryConfiguration directoryConfig;
directoryConfig = new DirectoryConfiguration()
{
Container = "profiles",
Path = profileStorage.RootPath,
//DirectoryQuotaInMB = 1000,
};
dmc.Directories.DataSources.Add(directoryConfig);
dmc.Directories.ScheduledTransferPeriod = TimeSpan.FromMinutes(1.0);
ridm.SetCurrentConfiguration(dmc);
}
public void DirectoryConfiguration_DefaultInitialisation_PropertyIconIsNull()
{
DirectoryConfiguration directoryConfiguration = new DirectoryConfiguration();
Assert.AreEqual(null, directoryConfiguration.Icon);
}
static void Main(string[] args)
{
string configXmlDir = Environment.CurrentDirectory;
if (args.Length == 1)
{
configXmlDir = args[0];
}
string configXmlPath = Path.Combine(configXmlDir, "DiagnosticsConfiguration.xml");
if (!File.Exists(configXmlPath))
{
throw new InvalidOperationException(string.Format("Unable to find diagnostics configuration xml @ {0}", configXmlPath));
}
XmlSerializer serializer = new XmlSerializer(typeof(DiagnosticsConfig));
DiagnosticsConfig diagConfig = null;
using (Stream fs = File.OpenRead(configXmlPath))
{
diagConfig = serializer.Deserialize(fs) as DiagnosticsConfig;
}
DiagnosticMonitorConfiguration config = DiagnosticMonitor.GetDefaultInitialConfiguration();
config.OverallQuotaInMB = diagConfig.OverallQuotaInMB;
if (diagConfig.Directories.Directory != null)
{
foreach (DiagnosticsConfigDirectoriesDirectory dir in diagConfig.Directories.Directory)
{
DirectoryConfiguration directoryConfig = new DirectoryConfiguration();
directoryConfig.Container = dir.ContainerName;
if (dir.IsLocalPathRelative)
{
directoryConfig.Path = Path.Combine(Environment.CurrentDirectory, dir.LocalPath);
}
else
{
directoryConfig.Path = dir.LocalPath;
}
directoryConfig.DirectoryQuotaInMB = dir.DirectoryQuotaInMB;
config.Directories.DataSources.Add(directoryConfig);
}
config.Directories.ScheduledTransferPeriod = TimeSpan.FromSeconds(diagConfig.Directories.ScheduledTransferPeriodInSeconds);
}
if (diagConfig.PerformanceCounters.PerformanceCounter != null)
{
foreach (DiagnosticsConfigPerformanceCountersPerformanceCounter perf in diagConfig.PerformanceCounters.PerformanceCounter)
{
PerformanceCounterConfiguration perfConfig = new PerformanceCounterConfiguration();
perfConfig.CounterSpecifier = perf.CounterName;
perfConfig.SampleRate = TimeSpan.FromSeconds(perf.SamplingRateInSeconds);
config.PerformanceCounters.DataSources.Add(perfConfig);
}
config.PerformanceCounters.ScheduledTransferPeriod = TimeSpan.FromSeconds(diagConfig.PerformanceCounters.ScheduledTransferPeriodInSeconds);
}
DiagnosticMonitor.Start(CloudStorageAccount.Parse(diagConfig.StorageAccountConnectionString), config);
Trace.WriteLine("ConfigureAzureDiagnostics Started.", "Information");
Console.ReadLine();
}
public void DirectoryConfiguration_Initialisation_PropertyFontIsInitialised()
{
DirectoryConfiguration directoryConfiguration = CreateDirectoryConfiguration();
Assert.AreEqual(true, directoryConfiguration.Font is DirectoryFontConfiguration);
}
public unsafe void RecoverAndTest(Guid cprVersion, Guid indexVersion)
{
// Recover
fht.Recover(cprVersion, indexVersion);
// Create array for reading
Empty context;
var inputArray = new Input[numUniqueKeys];
for (int i = 0; i < numUniqueKeys; i++)
{
inputArray[i].adId.adId = i;
inputArray[i].numClicks.numClicks = 0;
}
// Register with thread
fht.StartSession();
// Issue read requests
fixed(Input *input = inputArray)
{
for (var i = 0; i < numUniqueKeys; i++)
{
fht.Read(&((input + i)->adId), null, (Output *)&((input + i)->numClicks), &context, i);
}
}
// Complete all pending requests
fht.CompletePending(true);
// Release
fht.StopSession();
// Test outputs
Config.CheckpointDirectory = test_path;
var recoveryInfo = default(HybridLogRecoveryInfo);
recoveryInfo.Recover(cprVersion);
int num_threads = recoveryInfo.numThreads;
DirectoryInfo info = new DirectoryInfo(DirectoryConfiguration.GetHybridLogCheckpointFolder(cprVersion));
List <ExecutionContext> cpr_points = new List <ExecutionContext>();
foreach (var file in info.GetFiles())
{
if (file.Name != "info.dat" && file.Name != "snapshot.dat")
{
using (var reader = new StreamReader(file.FullName))
{
var ctx = new ExecutionContext();
ctx.Load(reader);
cpr_points.Add(ctx);
}
}
}
// Compute expected array
long[] expected = new long[numUniqueKeys];
long[] found = new long[numUniqueKeys];
long sno = cpr_points.First().serialNum;
for (long i = 0; i <= sno; i++)
{
var id = i % numUniqueKeys;
expected[id]++;
}
// Assert if expected is same as found
for (long i = 0; i < numUniqueKeys; i++)
{
Assert.IsTrue(
expected[i] == inputArray[i].numClicks.numClicks,
"Debug error for AdId {0}: Expected ({1}), Found({2})", inputArray[i].adId.adId, expected[i], inputArray[i].numClicks.numClicks);
}
}
public void DirectoryConfiguration_Initialisation_PropertyIconIsInitialised()
{
DirectoryConfiguration directoryConfiguration = CreateDirectoryConfiguration();
Assert.AreEqual(true, directoryConfiguration.Icon is DirectoryIconConfiguration);
}
