public void ConfigureServices(IServiceCollection services)
{
services.AddControllersWithViews();
services.AddSignalR();
SnLog.Instance = new SnFileSystemEventLogger();
SnTrace.SnTracers.Add(new SnFileSystemTracer());
SnTrace.EnableAll();
services.Configure <TaskManagementConfiguration>(Configuration.GetSection("TaskManagement"));
//TODO: inject allowed origins dynamically (do not allow everything)
services.AddCors(c =>
{
c.AddPolicy("AllowAllOrigins", options =>
{
options.AllowAnyOrigin();
options.AllowAnyHeader();
options.AllowAnyMethod();
});
});
services.AddSingleton <TaskDataHandler>();
services.AddSenseNetClientTokenStore();
services.AddSingleton <ApplicationHandler>();
services.AddHostedService <DeadTaskHostedService>();
}
private static void LogLine(string msg, EventType eventType)
{
switch (eventType)
{
case EventType.Error:
SnLog.WriteError(msg, categories: AdSyncLogCategory);
break;
case EventType.Warning:
SnLog.WriteWarning(msg, categories: AdSyncLogCategory);
break;
case EventType.Info:
SnLog.WriteInformation(msg, categories: AdSyncLogCategory);
break;
case EventType.Verbose:
SnTrace.Write("{0}: {1}", AdSync, msg);
break;
}
Console.WriteLine(msg);
// log event for subscriber of the current thread
StringBuilder sb;
if (Subscribers.TryGetValue(Thread.CurrentThread.GetHashCode(), out sb))
{
if (sb != null)
{
sb.AppendLine(GetMsgWithTimeStamp(msg));
}
}
}
internal static void Shutdown()
{
if (_instance == null)
{
_started = false;
SnLog.WriteWarning("Repository shutdown has already completed.");
return;
}
lock (_startStopSync)
{
if (_instance == null)
{
_started = false;
SnLog.WriteWarning("Repository shutdown has already completed.");
return;
}
SnTrace.Repository.Write("Sending a goodbye message.");
_instance.ConsoleWriteLine();
_instance.ConsoleWriteLine("Sending a goodbye message...");
DistributedApplication.ClusterChannel.ClusterMemberInfo.NeedToRecover = false;
var pingMessage = new PingMessage();
pingMessage.SendAsync(CancellationToken.None).GetAwaiter().GetResult();
foreach (var svc in _instance.serviceInstances)
{
SnTrace.Repository.Write("Shutting down {0}", svc.GetType().Name);
svc.Shutdown();
}
SnTrace.Repository.Write("Shutting down {0}", DistributedApplication.ClusterChannel.GetType().Name);
DistributedApplication.ClusterChannel.ShutDownAsync(CancellationToken.None).GetAwaiter().GetResult();
SnTrace.Repository.Write("Shutting down Security.");
SecurityHandler.ShutDownSecurity();
SnTrace.Repository.Write("Shutting down IndexingEngine.");
IndexManager.ShutDown();
ContextHandler.Reset();
var t = DateTime.UtcNow - _instance._startupInfo.Starting;
var msg = $"Repository has stopped. Running time: {t.Days}.{t.Hours:d2}:{t.Minutes:d2}:{t.Seconds:d2}";
SnTrace.Repository.Write(msg);
SnTrace.Flush();
_instance.ConsoleWriteLine(msg);
_instance.ConsoleWriteLine();
SnLog.WriteInformation(msg);
_instance = null;
_started = false;
}
}
public async Task <T> ExecuteReaderAsync <T>(string script, Action <DbCommand> setParams,
Func <DbDataReader, CancellationToken, Task <T> > callbackAsync)
{
using (var op = SnTrace.Database.StartOperation(GetOperationMessage("ExecuteReaderAsync", script)))
{
try
{
using (var cmd = CreateCommand())
{
cmd.Connection = OpenConnection();
cmd.CommandTimeout = Configuration.Data.DbCommandTimeout;
cmd.CommandText = script;
cmd.CommandType = CommandType.Text;
cmd.Transaction = _transaction?.Transaction;
setParams?.Invoke(cmd);
var cancellationToken = _transaction?.CancellationToken ?? _cancellationToken;
using (var reader = (DbDataReader)await cmd.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false))
{
cancellationToken.ThrowIfCancellationRequested();
var result = await callbackAsync(reader, cancellationToken).ConfigureAwait(false);
op.Successful = true;
return(result);
}
}
}
catch (Exception e)
{
SnTrace.WriteError(e.ToString());
throw;
}
}
}
public void SnTrace_DynamicCategories()
{
// Test initialization
CleanupAndEnableAll();
// Activate Custom and Test categories. Any other be inactive.
SnTrace.DisableAll();
SnTrace.Custom.Enabled = true;
SnTrace.Test.Enabled = true;
// Write 7 lines including dynamic categories
SnTrace.Write("Line1");
SnTrace.Category("asdf").Write("Line2");
SnTrace.Test.Write("Line3");
SnTrace.Category("qwer").Write("Line4");
SnTrace.Test.Write("Line5");
SnTrace.Category("yxcv").Write("Line6");
SnTrace.Write("Line7");
// Get log
var log = DisableAllAndGetLog();
// Get categories
var categories = log
.Select(Entry.Parse)
.Where(e => e != null)
.Select(e => e.Category)
.ToArray();
var actual = string.Join(",", categories);
// Verify
Assert.AreEqual("Custom,asdf,Test,qwer,Test,yxcv,Custom", actual);
}
public static void Main(string[] args)
{
var builder = CreateWebHostBuilder(args);
var host = builder.Build();
SnTrace.EnableAll();
using (InMemoryExtensions.StartInMemoryRepository(repositoryBuilder =>
{
repositoryBuilder
.UseAccessProvider(new UserAccessProvider())
.UseLogger(new SnFileSystemEventLogger())
.UseTracer(new SnFileSystemTracer());
}))
{
using (new SystemAccount())
{
// FOR TESTING PURPOSES: create a default user with a well-known password
// login: [email protected]
// password: Edvin123%
var parentPath = "/Root/IMS/BuiltIn/Temp";
var parent = RepositoryTools.CreateStructure(parentPath, "OrganizationalUnit");
var user = new User(parent.ContentHandler)
{
Name = "edvin-example.com",
LoginName = "*****@*****.**",
PasswordHash =
"AQAAAAEAACcQAAAAEKEsynr6baKE5rYqS4Rn6pjqckl+NG4W9UQqqGh4g23zlJQpQvnaZnzx44+z78FVsg==",
Email = "*****@*****.**"
};
user.Save();
// set the new user as administrator
Group.Administrators.AddMember(user);
// create a container for test content
parent = RepositoryTools.CreateStructure("/Root/MyContent", "SystemFolder");
// create a doclib that contains a file
var docLib = RepositoryTools.CreateStructure("/Root/MyContent/MyFiles", "DocumentLibrary");
((GenericContent)docLib.ContentHandler).AllowChildType("Image", save: true);
var file = new File(docLib.ContentHandler)
{
Name = "testfile.txt"
};
file.Binary.SetStream(RepositoryTools.GetStreamFromString($"temp text data {DateTime.UtcNow}"));
file.Save();
//var installer = new Installer();
//installer.Import("C:\\temp\\import\\Root");
}
SnTrace.EnableAll();
host.Run();
}
}
internal bool IsFeatureEnabled(int id)
{
if (!__isFeatureEnabled)
{
SnTrace.Write($"EventDistributor INACTIVATED ({id}).");
}
return(__isFeatureEnabled);
}
public void WriteLock_DeletionAfterKilledProcess()
{
var tracer = GetTracer();
SnTrace.EnableAll();
// Ensure an existing but free index directory
SnTrace.Write("1 creating index");
var indexDir = EnsureIndexDirectoryAsync().ConfigureAwait(false).GetAwaiter().GetResult();
SnTrace.Write("2 index created");
// Start a process that use the directory
var exePath = Path.GetFullPath(Path.Combine(
System.Reflection.Assembly.GetExecutingAssembly().Location,
"..\\..\\..\\..\\WorkerForWriteLockDeletionTests\\bin\\Debug\\netcoreapp3.1\\WorkerForWriteLockDeletionTests.exe"));
var process = Process.Start(exePath, $"100000 {indexDir.Name} {indexDir.Path}");
if (process == null)
{
Assert.Fail("Cannot start the process.");
}
SnTrace.Write("3 process started");
// Wait for the indexing engine uses the index
while (!File.Exists(indexDir.LockPath))
{
Task.Delay(100).ConfigureAwait(false).GetAwaiter().GetResult();
}
SnTrace.Write("4 index locked");
// Start the new indexing engine in async way.
SnTrace.Write("5 starting new engine");
Task.Run(() => { StartNewEngineAsync(indexDir.Name, indexDir.Path).ConfigureAwait(false); });
SnTrace.Write("6 wait a bit");
// Wait a bit
Task.Delay(1000).ConfigureAwait(false).GetAwaiter().GetResult();
// Check the console: the new engine cannot delete the write.lock file
var lastLine1 = tracer.Lines[tracer.Lines.Count - 1];
// Kill the lock owner process
SnTrace.Write("7 killing the process");
process.Kill();
//Task.Delay(20).ConfigureAwait(false).GetAwaiter().GetResult();
//File.Delete(indexDir.LockPath);
SnTrace.Write("8 wait a bit");
// lock file remains but deletable
Task.Delay(2000).ConfigureAwait(false).GetAwaiter().GetResult();
SnTrace.Write("9 test finished");
// Check the console: the new engine has started
Assert.IsTrue(tracer.Lines.Any(x => x.EndsWith("101 started")));
}
public void _FinishTest(TestContext testContext)
{
if (_snTraceOperation != null)
{
_snTraceOperation.Successful = true;
_snTraceOperation.Dispose();
}
SnTrace.Flush();
}
public void SnTrace_WriteEmpty_WithParams()
{
CleanupAndEnableAll();
SnTrace.Write(null, 1, "asdf");
var log = DisableAllAndGetLog();
AssertOneErrorLine(log, "Value cannot be null");
}
public void SnTrace_WriteNotEmpty_NullParams()
{
CleanupAndEnableAll();
SnTrace.Write("asdf: {0}", null);
var log = DisableAllAndGetLog();
AssertOneErrorLine(log, "asdf: {0}");
}
public void SnTrace_WriteEmpty_WithoutParams()
{
CleanupAndEnableAll();
SnTrace.Write(null);
var log = DisableAllAndGetLog();
AssertOneErrorLine(log, "SnTrace_WriteEmpty_WithoutParams");
}
public void SnTrace_SmartFormat_IntList()
{
CleanupAndEnableAll();
SnTrace.Write("asdf: {0}", new List <int>(new[] { 1, 2, 3, 4 }));
var log = DisableAllAndGetLog();
var msg = GetMessageFromLine(log[0]);
Assert.AreEqual("asdf: [1, 2, 3, 4]", msg);
}
public void SnTrace_SmartFormat_StringList()
{
CleanupAndEnableAll();
SnTrace.Write("asdf: {0}", new List <string>(new[] { "asdf", "qwer", "yxcv" }));
var log = DisableAllAndGetLog();
var msg = GetMessageFromLine(log[0]);
Assert.AreEqual("asdf: [asdf, qwer, yxcv]", msg);
}
public void SnTrace_WriteNotEmpty_NullValue()
{
CleanupAndEnableAll();
SnTrace.Write("asdf: {0}, {1}, {2}", 42, null, "asdf");
var log = DisableAllAndGetLog();
var msg = GetMessageFromLine(log[0]);
Assert.AreEqual("asdf: 42, [null], asdf", msg);
}
protected virtual void Application_End(object sender, EventArgs e, HttpApplication application)
{
using (var op = SnTrace.Repository.StartOperation("Application_End"))
{
Repository.Shutdown();
SnLog.WriteInformation("Application_End", EventId.RepositoryLifecycle, properties: new Dictionary <string, object> {
{ "ShutdownReason", HostingEnvironment.ShutdownReason }
});
op.Successful = true;
}
SnTrace.Flush();
}
public void SnTrace_WriteError()
{
CleanupAndEnableAll();
SnTrace.WriteError("asdf");
var log = DisableAllAndGetLog();
Assert.AreEqual(1, log.Count);
Assert.AreEqual("ERROR", GetColumnFromLine(log[0], 6));
Assert.IsTrue(log[0].EndsWith("asdf"));
}
public void SnTrace_SmartFormat_LinqExpressionWhenDisabled()
{
CleanupAndEnableAll();
_expressionExecuted = false;
SnTrace.Custom.Enabled = false;
SnTrace.Write("asdf: {0}", Filter(Enumerable.Range(40, 5)));
var log = DisableAllAndGetLog();
Assert.AreEqual(0, log.Count);
Assert.IsFalse(_expressionExecuted);
}
public void SnTrace_Write2lines()
{
CleanupAndEnableAll();
SnTrace.Write("asdf");
SnTrace.Write("qwer");
var log = DisableAllAndGetLog();
Assert.AreEqual(2, log.Count);
Assert.IsTrue(log[0].EndsWith("asdf"));
Assert.IsTrue(log[1].EndsWith("qwer"));
}
public void CleanupTest()
{
SnTrace.Test.Enabled = true;
//SnTrace.Test.Write("END test: {0}", TestContext.TestName);
if (_testMethodOperation != null)
{
_testMethodOperation.Successful = true;
_testMethodOperation.Dispose();
}
SnTrace.Flush();
}
public void CleanupTest()
{
SnTrace.Test.Enabled = true;
SnTrace.Test.Write("{0}: {1}", TestContext.TestName, TestContext.CurrentTestOutcome);
if (_testMethodOperation != null)
{
_testMethodOperation.Successful = true;
_testMethodOperation.Dispose();
}
SnTrace.Flush();
}
// ReSharper disable once UnusedParameter.Local
static async Task Main(string[] args)
{
IConfiguration config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", true, true)
.Build();
config.GetSection("TaskManagement").Bind(AgentConfig);
SnLog.Instance = new SnFileSystemEventLogger();
SnTrace.SnTracers.Add(new SnFileSystemTracer());
SnTrace.EnableAll();
AgentName = AgentManager.GetAgentName();
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
try
{
DiscoverCapabilities();
_updateLockTimer = new Timer(UpdateLockTimerElapsed);
_watchExecutorTimer = new Timer(WatchExecutorTimerElapsed);
var started = await StartSignalR();
_heartBeatTimerPeriodInMilliseconds = AgentConfig.HeartbeatPeriodInSeconds * 1000;
_heartbeatTimer = new Timer(HeartBeatTimerElapsed, null, _heartBeatTimerPeriodInMilliseconds, _heartBeatTimerPeriodInMilliseconds);
#pragma warning disable 4014
// start processing on a background thread
if (started)
{
WorkAsync();
}
#pragma warning restore 4014
Console.ReadLine();
if (_hubConnection != null)
{
await _hubConnection.DisposeAsync();
}
}
catch (Exception ex)
{
SnLog.WriteException(ex, string.Empty, EventId.TaskManagement.General);
}
_heartbeatTimer?.Change(Timeout.Infinite, Timeout.Infinite);
_heartbeatTimer?.Dispose();
}
public void SnTrace_DynamicCategoryOnOff()
{
// Test initialization
CleanupAndEnableAll();
// Activate Custom and Test categories. Any other be inactive.
SnTrace.DisableAll();
SnTrace.Custom.Enabled = true;
SnTrace.Test.Enabled = true;
// Pin custom categories
var asdf = SnTrace.Category("asdf");
var qwer = SnTrace.Category("qwer");
var yxcv = SnTrace.Category("yxcv");
asdf.Write("0");
qwer.Write("1");
yxcv.Write("2");
asdf.Enabled = false;
asdf.Write("3");
qwer.Write("4");
yxcv.Write("5");
yxcv.Enabled = false;
asdf.Write("6");
qwer.Write("7");
yxcv.Write("8");
asdf.Enabled = true;
yxcv.Enabled = true;
qwer.Enabled = false;
asdf.Write("9");
qwer.Write("A");
yxcv.Write("B");
qwer.Enabled = true;
asdf.Write("C");
qwer.Write("D");
yxcv.Write("E");
// Get log
var log = DisableAllAndGetLog();
// Get categories
var categories = log
.Select(Entry.Parse)
.Where(e => e != null)
.Select(e => e.Message)
.ToArray();
var actual = string.Join("", categories);
// Verify
Assert.AreEqual("0124579BCDE", actual);
}
public void SnTrace_SmartFormat_LinqExpressionWhenEnabled()
{
CleanupAndEnableAll();
_expressionExecuted = false;
SnTrace.Write("asdf: {0}", Filter(Enumerable.Range(40, 5)));
var log = DisableAllAndGetLog();
var msg = GetMessageFromLine(log[0]);
Assert.AreEqual(1, log.Count);
Assert.IsTrue(_expressionExecuted);
Assert.AreEqual("asdf: [41, 42, 43]", msg);
}
public void _StartTest(TestContext testContext)
{
//if (!SnTrace.SnTracers.Any(x => x is SnDebugViewTracer))
// SnTrace.SnTracers.Add(new SnDebugViewTracer());
if (!SnTrace.SnTracers.Any(x => x is SnFileSystemTracer))
{
SnTrace.SnTracers.Add(new SnFileSystemTracer());
}
SnTrace.EnableAll();
SnTrace.SecurityQueue.Enabled = false;
_snTraceOperation =
SnTrace.Test.StartOperation(
$"TESTMETHOD: {testContext.FullyQualifiedTestClassName}.{testContext.TestName}");
}
private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null)
{
server ??= ClientContext.Current.Server;
// force set values
uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes;
if (uploadData.FileLength == 0)
{
uploadData.FileLength = binaryStream.Length;
}
requestData.Parameters.Add("create", "1");
dynamic uploadedContent = null;
// Get ChunkToken
try
{
SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", requestData);
var retryCount = 0;
await Retrier.RetryAsync(10, 1000, async() =>
{
retryCount++;
var retryText = retryCount > 1 ? $" (retry {retryCount})" : string.Empty;
server.Logger?.LogTrace($"Uploading initial data of {uploadData.FileName}{retryText}.");
var httpContent = new StringContent(uploadData.ToString());
httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType);
await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent,
response =>
{
uploadData.ChunkToken = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
}, CancellationToken.None).ConfigureAwait(false);
}, (i, exception) =>
{
// choose the exceptions when we can retry the operation
return(exception switch
{
null => true,
ClientException cex when
(int) cex.StatusCode == 429 ||
cex.ErrorData?.ExceptionType == "NodeIsOutOfDateException"
=> false,
_ => throw exception
});
});
/// <summary>
/// Enables trace categories. It only switches the provided categories ON,
/// it does not switch off the ones that are not listed.
/// </summary>
internal static void UpdateCategories(string[] categoryNames)
{
if (categoryNames == null)
{
SnTrace.DisableAll();
return;
}
// do not switch off any category, only switch ON the listed ones
foreach (var category in SnTrace.Categories.Where(c => categoryNames.Contains(c.Name)))
{
category.Enabled = true;
}
SnTrace.System.Write("Trace settings were updated. Enabled: {0}", string.Join(", ", SnTrace.Categories
.Where(c => c.Enabled).Select(c => c.Name)));
}
private static async Task <bool> InitializeAsync()
{
ServiceProvider = new ServiceCollection()
.AddLogging()
.AddSenseNetClientTokenStore()
.BuildServiceProvider();
SnTrace.EnableAll();
Configuration.Initialize();
SnTrace.SnTracers.Add(new SnFileSystemTracer());
ServicePointManager.DefaultConnectionLimit = 10;
ClientContext.Current.ChunkSizeInBytes = Config.Upload.ChunkSize;
ClientContext.Current.AddServer(new ServerContext
{
Url = SiteUrl,
IsTrusted = Config.Environment.IsDevelopment
});
try
{
var tokenStore = ServiceProvider.GetRequiredService <TokenStore>();
var secret = string.IsNullOrEmpty(Password) ? "secret" : Password;
var token = await tokenStore.GetTokenAsync(ClientContext.Current.Server, secret);
ClientContext.Current.Server.Authentication.AccessToken = token;
if (string.IsNullOrEmpty(token))
{
SnTrace.System.Write("Access token is empty, fallback to user name and password.");
ClientContext.Current.Server.Username = Username;
ClientContext.Current.Server.Password = Password;
}
}
catch (Exception ex)
{
Logger.WriteError(ContentId, 0, ex: ex, startIndex: StartIndex, version: Version, message:
$"Authentication failed for site {SiteUrl}: {ex.Message}");
return(false);
}
return(true);
}
public void SnTrace_Entries_3lines()
{
CleanupAndEnableAll();
SnTrace.Write("asdf asdf");
SnTrace.Write("qwer\nqwer");
SnTrace.Write("yxcv\tyxcv");
var log = DisableAllAndGetLog();
var entries = log.Select(Entry.Parse).Where(e => e != null).ToArray();
Assert.AreEqual(3, entries.Length);
Assert.AreEqual("asdf asdf", entries[0].Message);
Assert.AreEqual("qwer.qwer", entries[1].Message);
Assert.AreEqual("yxcv\tyxcv", entries[2].Message);
}
private async Task StartNewEngineAsync(string name, string path)
{
var sb = new StringBuilder();
var console = new StringWriter(sb);
var indexDirectory = new IndexDirectory(name, path);
var engine = new Lucene29LocalIndexingEngine(indexDirectory);
SnTrace.Write("100 starting");
await engine.StartAsync(console, false, CancellationToken.None).ConfigureAwait(false);
SnTrace.Write("101 started");
while (!File.Exists(indexDirectory.IndexLockFilePath))
{
Task.Delay(50).ConfigureAwait(false).GetAwaiter().GetResult();
}
await engine.ShutDownAsync(CancellationToken.None).ConfigureAwait(false);
}