static void BUDDY_BUG(string msg, uint value = 0)
{
//printf("BUDDY_BUG in %s, %d.\n", f, line);
//System.Console.WriteLine(msg);
KernelMessage.Path("Allocator", msg, value);
//assert(0);
}
public Kernel(ConnectionInformation connectionInformation)
{
this.connectionInformation = connectionInformation;
// heartbeat
hbSocket = new RouterSocket();
hbSocket.Bind($"{connectionInformation.transport}://{connectionInformation.ip}:{connectionInformation.hb_port}");
// control
controlSocket = new RouterSocket();
controlSocket.Bind($"{connectionInformation.transport}://{connectionInformation.ip}:{connectionInformation.control_port}");
// stdin
stdinSocket = new RouterSocket();
stdinSocket.Bind($"{connectionInformation.transport}://{connectionInformation.ip}:{connectionInformation.stdin_port}");
// iopub
ioSocket = new PublisherSocket();
ioSocket.Bind($"{connectionInformation.transport}://{connectionInformation.ip}:{connectionInformation.iopub_port}");
// shell
shellSocket = new RouterSocket();
shellSocket.Bind($"{connectionInformation.transport}://{connectionInformation.ip}:{connectionInformation.shell_port}");
payload = new List <Payload>();
nugetManager = new NuGetManager(new FileInfo(".").FullName);
executionCount = 0;
lastMessage = null;
headerCode = getHeaderCode();
hmac = new HMACSHA256(Encoding.UTF8.GetBytes(connectionInformation.key));
}
private void PrintAddress()
{
KernelMessage.WriteLine("PageDirectoryPTPhys: {0:X8}", this.GetPageTablePhysAddr());
KernelMessage.WriteLine("PageDirectoryPT: {0:X8}", AddrPageDirectoryPT);
KernelMessage.WriteLine("PageDirectory: {0:X8}", AddrPageDirectory);
KernelMessage.WriteLine("PageTable: {0:X8}", (uint)PageTableEntries);
}
public void intellisenseRequest(KernelMessage msg, IntellisenseRequest content)
{
var codeCells = JsonConvert.DeserializeObject <List <String> >(content.text);
codeCells.Add(headerCode);
var position = JsonConvert.DeserializeObject <BlockType>(content.block);
var lineOffset = 0;
foreach (var line in codeCells.Take(position.selectedIndex + 1))
{
lineOffset += line.Split('\n').Length;
}
var realLineNumber = position.line + lineOffset + 1;
var codeString = string.Join("\n", codeCells);
// TODO: Finish implementing intellisense...
// Evaluation.GetDeclarations(codeString, realLineNumber, position.ch);
var newContent = new CompleteReply()
{
matched_text = "matched text",
filter_start_index = 0,
matches = new object(),
status = "ok"
};
sendDisplayData("errors", new object(), "display_data");
sendMessage(shellSocket, msg, "complete_reply", newContent);
}
private static void MapKernelImage()
{
var phys = Address.KernelElfSectionPhys;
var diff = Address.KernelBaseVirt - Address.KernelBasePhys;
var endPhys = phys + OriginalKernelElf.TotalFileSize;
var addr = phys;
KernelMessage.WriteLine("Mapping Kernel Image from physical {0:X8} to virtual {1:X8}", phys, phys + diff);
while (addr < endPhys)
{
PageTable.KernelTable.MapVirtualAddressToPhysical(addr + diff, addr);
addr += 0x1000;
}
PageTable.KernelTable.Flush();
var map = new BootInfoMemory
{
Start = phys + diff,
Size = OriginalKernelElf.TotalFileSize,
Type = BootInfoMemoryType.KernelElfVirt,
AddressSpaceKind = AddressSpaceKind.Virtual,
};
BootInfo_.AddMap(map);
}
public static void Dump(bool canScheduled)
{
for (var i = 0; i < ThreadCapacity; i++)
{
var th = Threads[i];
if (th.CanScheduled == canScheduled && th.Status != ThreadStatus.Empty)
{
KernelMessage.Write("ThreadID={0} Status={1}", th.ThreadID, (uint)th.Status);
if (th.DebugName != null)
{
KernelMessage.Write(" DebugName=" + th.DebugName);
}
if (th.Process != null && th.Process.Path != null)
{
KernelMessage.Write(" proc=" + th.Process.Path);
}
if (th.Process != null)
{
KernelMessage.Write(" procID=" + th.Process.ProcessID);
}
if (th.DebugSystemMessage.Target > 0)
{
var msg = th.DebugSystemMessage;
KernelMessage.Write(" Target={0} arg1={1} arg2={2} arg3={3}", (uint)msg.Target, msg.Arg1, msg.Arg2, msg.Arg3);
}
KernelMessage.Write('\n');
}
}
}
public virtual void Setup(MemoryRegion region, AddressSpaceKind addressSpaceKind)
{
AddressSpaceKind = addressSpaceKind;
region.Size = KMath.FloorToPowerOfTwo(region.Size);
_Region = region;
var totalPages = region.Size >> BuddyAllocatorImplementation.BUDDY_PAGE_SHIFT;
KernelMessage.WriteLine("Init Allocator: StartAddr: {0}, {1} Pages", region.Start, totalPages);
// init global memory block
// all pages area
var pages_size = totalPages * (uint)sizeof(Page);
KernelMessage.WriteLine("Page Array Size in bytes: {0}", pages_size);
Pages = (Page *)AllocRawMemory(pages_size).Start;
KernelMessage.WriteLine("Page Array Addr: {0:X8}", (uint)Pages);
var start_addr = region.Start;
Zone.free_area = (BuddyAllocatorImplementation.free_area *)AllocRawMemory(BuddyAllocatorImplementation.BUDDY_MAX_ORDER * (uint)sizeof(BuddyAllocatorImplementation.free_area)).Start;
fixed(BuddyAllocatorImplementation.mem_zone *zone = &Zone)
ZonePtr = zone;
BuddyAllocatorImplementation.buddy_system_init(
ZonePtr,
Pages,
start_addr,
totalPages);
}
public static void DumpStats()
{
KernelMessage.WriteLine("Threads, Can scheduled:");
Dump(true);
KernelMessage.WriteLine("Non-Schedulable Threads:");
Dump(false);
}
/// <summary>
/// Enable the IDTManager
/// </summary>
public static void Start()
{
KernelMessage.Write("Enabling interrupts...");
Enabled = true;
Flush();
KernelMessage.WriteLine("done");
}
internal static uint WriteDebugChar(ref SysCallContext context, ref SystemMessage args)
{
var c = (char)args.Arg1;
KernelMessage.Write(c);
return(0);
}
private static uint Cmd_WriteDebugChar(SysCallContext *context, SystemMessage *args)
{
var c = (char)args->Arg1;
KernelMessage.Write(c);
return(0);
}
public void kernelInfoRequest(KernelMessage msg, KernelRequest content)
{
var reply = new KernelReply()
{
protocol_version = "4.0.0",
implementation = "icsharp",
implementation_version = "4.0.0",
banner = "",
help_links = new KernelReply_HelpLink[] { },
language = "csharp",
language_info = new KernelReply_LanguageInfo()
{
name = "csharp",
version = "6.0.0.0",
mimetype = "text/x-csharp",
file_extension = ".cs",
pygments_lexer = "",
codemirror_mode = "",
nbconvert_exporter = ""
}
};
sendStateBusy(msg);
sendMessage(shellSocket, msg, "kernel_info_reply", reply);
}
public unsafe Page *AllocatePages(uint pages, AllocatePageOptions options = default)
{
if (pages > _FreePages)
{
Panic.Error("Out of Memory");
return(null);
}
Page *page;
if (AddressSpaceKind == AddressSpaceKind.Virtual || options.Continuous)
{
page = AllocatePagesContinuous(pages, options);
}
else
{
page = AllocatePagesNormal(pages, options);
}
if (page == null)
{
KernelMessage.WriteLine("DebugName: {0}", DebugName);
KernelMessage.WriteLine("Free pages: {0:X8}, Requested: {1:X8}", FreePages, pages);
Panic.Error("Out of Memory");
}
KernelMessage.Path(DebugName, "SimpleAlloc: Request {0} Pages, Addr {1:X8}", pages, GetAddress(page));
return(page);
}
public static unsafe void SaveThreadState(uint threadID, IntPtr stackState)
{
//Assert.True(threadID < MaxThreads, "SaveThreadState(): invalid thread id > max");
var thread = Threads[threadID];
if (thread.Status != ThreadStatus.Running)
{
return; // New threads doesn't have a stack in use. Take the initial one.
}
//Assert.True(thread != null, "SaveThreadState(): thread id = null");
if (thread.User)
{
Assert.IsSet(thread.StackState, "thread.StackState is null");
*thread.StackState = *(IDTTaskStack *)stackState;
}
else
{
thread.StackState = (IDTTaskStack *)stackState;
}
if (KConfig.Log.TaskSwitch)
{
KernelMessage.Write("Task {0}: Stored ThreadState from {1:X8} stored at {2:X8}, EIP={3:X8}", threadID, (uint)stackState, (uint)thread.StackState, thread.StackState->Stack.EIP);
if (thread.User)
{
KernelMessage.WriteLine(" ESP={0:X8}", thread.StackState->TASK_ESP);
}
else
{
KernelMessage.WriteLine();
}
}
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
ScrapySource scrapySource = await coreCache.RestoreAsync <ScrapySource>(PrefixConst.SOURCE_META + kernelMessage.JobId);
var sourceType = scrapySource.Source.Type;
IExtractor extractor = extractorManager.GetExtrator(sourceType);
await extractor.ExtractTarget(scrapySource.Source.Parameters.ToString(), scrapySource.SaveTo);
string transformJobIdsKey = PrefixConst.SOURCE_TRANSFOR_MAP + kernelMessage.JobId;
TaskingManager taskingManager = new TaskingManager();
List <string> jobIds = await coreCache.RestoreAsync <List <string> >(transformJobIdsKey);
if (jobIds != null)
{
foreach (var x in jobIds)
{
KernelMessage transforMsg = new KernelMessage()
{
JobId = x,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
};
taskingManager.AddTask(platformExit.OutRandom(transforMsg));
logger.Debug(JsonConvert.SerializeObject(transforMsg));
}
}
await taskingManager.WhenAll();
}
/// <summary>
/// Dump the statistics of interest
/// </summary>
public static unsafe void DumpStats()
{
for (var i = 0; i < ProcessList.Count; i++)
{
var proc = ProcessList[i];
KernelMessage.WriteLine("PID {1} Name {0} State {2}", proc.Path, (uint)proc.ProcessID, (uint)proc.RunState);
}
}
private static uint Cmd_WriteDebugMessage(SysCallContext *context, SystemMessage *args)
{
var msg = (NullTerminatedString *)args->Arg1;
KernelMessage.WriteLine(msg);
return(0);
}
private static void AssertError(string message, uint arg1 = 0, uint arg2 = 0, uint arg3 = 0)
{
KernelMessage.WriteLine("ASSERT ERROR! ");
var sb = new StringBuffer();
sb.Append(message, arg1, arg2, arg3);
KernelMessage.WriteLine(sb);
}
internal static unsafe uint WriteDebugMessage(ref SysCallContext context, ref SystemMessage args)
{
var msg = (NullTerminatedString *)args.Arg1;
KernelMessage.WriteLine(msg);
return(0);
}
private static void PrintMapArray(string name, KernelMemoryMapArray *mapArray)
{
KernelMessage.WriteLine("Items of MemoryMap Array [{0}]", name);
for (var i = 0; i < mapArray->Count; i++)
{
var mm = &mapArray->Items[i];
KernelMessage.WriteLine("Map {0:X8}-{1:X8}, Size={2:X8}, Type={3}, AddrKind={4}", mm->Start, mm->Start + mm->Size - 1, mm->Size, (uint)mm->Type, (uint)mm->AddressSpaceKind);
}
}
public void historyRequest(KernelMessage msg, HistoryRequest content)
{
var reply = new HistoryReply()
{
history = new List <string>()
};
sendMessage(shellSocket, msg, "history_reply", reply);
}
public void inspectRequest(KernelMessage msg, InspectRequest content)
{
var reply = new InspectReply()
{
status = "ok", found = false, data = new Dictionary <string, object>(), metadata = new Dictionary <string, object>()
};
sendMessage(shellSocket, msg, "inspect_reply", reply);
}
public void EnablePaging()
{
KernelMessage.Write("Enable Paging... ");
// Set CR0 register on processor - turns on virtual memory
Native.SetCR0(Native.GetCR0() | 0x80000000);
KernelMessage.WriteLine("Done");
}
public static void Setup()
{
KernelMessage.WriteLine("Setup ELF Headers");
//KernelMessage.WriteLine("Image Header:");
//KernelMemory.DumpToConsoleLine(Address.KernelElfSection, 124);
Main = FromAddress(Address.KernelElfSectionVirt);
Native = FromSectionName("native");
}
/// <summary>
/// Releases a page to the free list
/// </summary>
public void Free(Page *page)
{
var oldFree = _FreePages;
string debugName = null;
if (page->DebugTag != null)
{
debugName = (string)Intrinsic.GetObjectFromAddress((Pointer)(uint)page->DebugTag);
}
UninterruptableMonitor.Enter(this);
try
{
var debugCount = list_head.list_count((list_head *)page); // DEBUG
SelfCheck("SCF1", debugCount);
Page *temp = page;
uint result = 0;
do
{
result++;
if (temp->Status == PageStatus.Free)
{
//Panic.Error("Double Free");
SelfCheck("SCF3", debugCount);
KernelMessage.WriteLine("Double Free. Pages {0} Iteration {1}", debugCount, result);
Debug.Break();
}
temp->Status = PageStatus.Free;
var oldTemp = temp;
temp = (Page *)temp->Lru.next;
Native.Nop();
_FreePages++;
list_head.list_move_tail((list_head *)oldTemp, FreeList);
}while (temp != page && result != debugCount);
//list_head.list_headless_splice_tail((list_head*)page, FreeList);
SelfCheck("SCF2", debugCount);
}
finally
{
UninterruptableMonitor.Exit(this);
}
var freedPages = _FreePages - oldFree;
if (KConfig.Log.PageAllocation && TraceOptions.Enabled && freedPages >= TraceOptions.MinPages)
{
KernelMessage.Path(DebugName, "Freed Pages: {1}. Addr: {2:X8}. Now available: {3} --> {4}. Allocations={5} DebugName={0}.", debugName, freedPages, GetAddress(page), oldFree, _FreePages, (uint)Requests);
}
_Releases++;
}
internal static void Undefined(ref IDTStack stack)
{
var handler = IDTManager.Handlers[stack.Interrupt];
if (handler.NotifyUnhandled)
{
handler.NotifyUnhandled = false;
KernelMessage.WriteLine("Unhandled Interrupt {0}", stack.Interrupt);
}
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage =
JsonConvert.DeserializeObject <KernelMessage>(
Encoding.UTF8.GetString(processMessage));
ScheduleMessage scheduleMessage = await RestoreMeta(kernelMessage);
var scheduler = ScheduleManager.Manager
.GetScheduler(scheduleMessage.Scheduler);
}
public void Setup(MemoryRegion region, AddressSpaceKind addrKind)
{
TraceOptions = new PageFrameAllocatorTraceOptions();
_AddressSpaceKind = addrKind;
_Region = region;
FistPageNum = region.Start / PageSize;
_TotalPages = region.Size / PageSize;
kmap = AllocRawMemory(_TotalPages * (uint)sizeof(Page));
PageArray = (Page *)kmap.Start;
var firstSelfPageNum = KMath.DivFloor(kmap.Start, 4096);
var selfPages = KMath.DivFloor(kmap.Size, 4096);
KernelMessage.WriteLine("Page Frame Array allocated {0} pages, beginning with page {1} at {2:X8}", selfPages, firstSelfPageNum, (uint)PageArray);
PageTable.KernelTable.SetWritable(kmap.Start, kmap.Size);
kmap.Clear();
var addr = FistPageNum * 4096;
for (uint i = 0; i < _TotalPages; i++)
{
//KernelMessage.WriteLine(i);
PageArray[i].Address = addr;
//if (i != 0)
// PageArray[i - 1].Next = &PageArray[i];
addr += 4096;
}
KernelMessage.WriteLine("Setup free memory");
SetupFreeMemory();
KernelMessage.WriteLine("Build linked lists");
BuildLinkedLists();
KernelMessage.WriteLine("Build linked lists done");
_FreePages = 0;
for (uint i = 0; i < _TotalPages; i++)
{
if (PageArray[i].Status == PageStatus.Free)
{
_FreePages++;
}
}
//Assert.True(list_head.list_count(FreeList) == _FreePages, "list_head.list_count(FreeList) == _FreePages");
var debugCheckCount = list_head.list_count(FreeList);
if (debugCheckCount != _FreePages)
{
KernelMessage.WriteLine("debugCheckCount {0} != {1}", debugCheckCount, _FreePages);
Debug.Break();
}
KernelMessage.Path(DebugName, "Pages Free: {0}", FreePages);
}
public async Task Process(byte[] processMessage, IPlatformExit platformExit)
{
KernelMessage kernelMessage = JsonConvert.DeserializeObject <KernelMessage>(Encoding.UTF8.GetString(processMessage));
TransformEvent transformEvent = await CoreCache.RestoreAsync <TransformEvent>(PrefixConst.TRANSFORM_META + kernelMessage.JobId);
if (transformEvent != null)
{
try
{
TransformDataSet transformDataSet = new TransformDataSet();
string data = await coreStorage.GetStringAsync(transformEvent.GetFrom);
foreach (var def in transformEvent.FieldDefinitions)
{
ContextData contextData = new ContextData()
{
ContentText = data
};
var convertorSquence = def.ConvertorNavigators.Select(x => ConvertorManager.GetConvertor(x));
foreach (var convertor in convertorSquence)
{
contextData = convertor.Convert(contextData);
}
TransformFieldWithValue transformFieldWithValue = new TransformFieldWithValue();
PackageTransformFieldWithValue(transformFieldWithValue, contextData);
transformFieldWithValue.Name = def.Name;
transformFieldWithValue.Title = def.Title;
transformDataSet.FieldValues[def.Name] = transformFieldWithValue;
}
using (Stream serialzedStream = await transformDataSet.SerialzeToStream(transformEvent.ExportAs))
{
await coreStorage.WriteStream(
serialzedStream,
transformEvent.SaveTo);
}
string loadKey = await CoreCache.RestoreStringAsync(PrefixConst.TRANSFORM_LOAD_MAP + transformEvent.JobId);
await platformExit.OutRandom(new KernelMessage()
{
JobId = loadKey,
MessageId = kernelMessage.MessageId,
MessageName = kernelMessage.MessageName
});
}
catch (Exception ex)
{
logger.Warn(ex);
}
}
else
{
//TODO: Debug
logger.Debug("TransformEvent is Null:" + PrefixConst.TRANSFORM_META + kernelMessage.JobId);
}
}
public void shutdownRequest(KernelMessage msg, ShutdownRequest content)
{
logMessage("shutdown request");
var reply = new ShutdownReply()
{
restart = true
};
sendMessage(shellSocket, msg, "shutdown_reply", reply);
System.Environment.Exit(0);
}
