public virtual RetCode RemoveStateDesc(string stateDesc) { StringBuilder buf; RetCode rc; int p, q; if (string.IsNullOrWhiteSpace(stateDesc)) { rc = RetCode.InvalidArg; // PrintError goto Cleanup; } rc = RetCode.Success; p = StateDesc.IndexOf(stateDesc, StringComparison.OrdinalIgnoreCase); if (p != -1) { buf = new StringBuilder(Constants.BufSize); buf.Append(StateDesc); q = p + stateDesc.Length; if (!Char.IsWhiteSpace(buf[p])) { while (q < buf.Length && Char.IsWhiteSpace(buf[q])) { q++; } } buf.Remove(p, q - p); StateDesc = buf.ToString().Trim(); } Cleanup: return(rc); }
public virtual RetCode AddStateDesc(string stateDesc, bool dupAllowed = false) { StringBuilder buf; RetCode rc; int p; if (string.IsNullOrWhiteSpace(stateDesc)) { rc = RetCode.InvalidArg; // PrintError goto Cleanup; } rc = RetCode.Success; p = StateDesc.IndexOf(stateDesc, StringComparison.OrdinalIgnoreCase); if (dupAllowed || p == -1) { buf = new StringBuilder(Constants.BufSize); buf.AppendFormat ( "{0}{1}{2}", StateDesc, StateDesc.Length > 0 ? " " : "", stateDesc ); StateDesc = buf.ToString(); } Cleanup: return(rc); }
public void ReadTxt(string path, List <string> sharedStrings = null) { // Two cases: either a single .edd.txt file, or chunks to join together and rely on shared strings. bool strRef = sharedStrings != null; order = sharedStrings ?? new List <string>(); if (strRef) { path = path.Replace(".edd.txt", "_part.edd.txt"); } Dictionary <int, MachineDesc> machines = new Dictionary <int, MachineDesc>(); foreach (string line in File.ReadLines(path)) { if (line.Length == 0) { continue; } string[] parts = line.Split(new[] { ' ' }, 2); if (parts.Length != 2 || !parts[0].EndsWith(":") || parts[1].StartsWith("#") != strRef) { throw new Exception($"{path}: bad line: {line}"); } string desc = parts[0].Substring(0, parts[0].Length - 1); if (strRef) { locs[desc] = int.Parse(parts[1].Substring(1)); } else { addStr(desc, parts[1]); } string name = order[locs[desc]]; parts = desc.Split('_'); if (!Common.ParseMachine(parts[0], out int machineId)) { throw new Exception($"Bad machine id in {desc}"); } if (!machines.TryGetValue(machineId, out MachineDesc machine)) { machine = machines[machineId] = new MachineDesc(machineId); } if (parts.Length == 1) { machine.Name = name; } else { int i = int.Parse(parts[1].Substring(1)); if (parts[1].StartsWith("p")) { machine.ParamNames[i] = name; } else if (parts[1].StartsWith("s")) { StateDesc state = machine.States.Find(s => s.ID == i); if (state == null) { state = new StateDesc(i); machine.States.Add(state); } if (parts.Length == 2) { state.Name = name; } else { // Ignore indices, add in order if (parts[2].StartsWith("e")) { state.EntryCommands.Add(new CommandDesc(name)); } else if (parts[2].StartsWith("x")) { state.ExitCommands.Add(new CommandDesc(name)); } else if (parts[2].StartsWith("w")) { state.WhileCommands.Add(new CommandDesc(name)); } } } else { throw new Exception(desc); } } } edd = new EDD(); edd.Machines = machines.Values.ToList(); }
internal override void Read(BinaryReaderEx br) { br.BigEndian = false; string magic = br.AssertASCII("fSSL", "fsSL"); LongFormat = magic == "fsSL"; br.VarintLong = LongFormat; br.AssertInt32(1); br.AssertInt32(1); br.AssertInt32(1); br.AssertInt32(0x7C); int dataSize = br.ReadInt32(); br.AssertInt32(11); br.AssertInt32(LongFormat ? 0x58 : 0x34); br.AssertInt32(1); br.AssertInt32(LongFormat ? 0x10 : 8); int stringCount = br.ReadInt32(); br.AssertInt32(4); br.AssertInt32(0); br.AssertInt32(8); int functionSpecCount = br.ReadInt32(); int conditionSize = br.AssertInt32(LongFormat ? 0x10 : 8); int conditionCount = br.ReadInt32(); br.AssertInt32(LongFormat ? 0x10 : 8); br.AssertInt32(0); br.AssertInt32(LongFormat ? 0x18 : 0x10); int commandSpecCount = br.ReadInt32(); int commandSize = br.AssertInt32(4); int commandCount = br.ReadInt32(); int passCommandSize = br.AssertInt32(LongFormat ? 0x10 : 8); int passCommandCount = br.ReadInt32(); int stateSize = br.AssertInt32(LongFormat ? 0x78 : 0x3C); int stateCount = br.ReadInt32(); br.AssertInt32(LongFormat ? 0x48 : 0x30); int machineCount = br.ReadInt32(); int stringsOffset = br.ReadInt32(); br.AssertInt32(0); br.AssertInt32(stringsOffset); Unk80 = br.ReadInt32(); br.AssertInt32(dataSize); br.AssertInt32(0); br.AssertInt32(dataSize); br.AssertInt32(0); long dataStart = br.Position; br.AssertVarint(0); long commandSpecOffset = br.ReadVarint(); br.AssertVarint(commandSpecCount); long functionSpecOffset = br.ReadVarint(); br.AssertVarint(functionSpecCount); long machineOffset = br.ReadVarint(); br.AssertInt32(machineCount); UnkB0 = br.ReadInt32s(4); if (LongFormat) { br.AssertInt32(0); } br.AssertVarint(LongFormat ? 0x58 : 0x34); br.AssertVarint(stringCount); List <string> strings = new List <string>(); for (int i = 0; i < stringCount; i++) { long stringOffset = br.ReadVarint(); // Char count not needed as all strings are null-terminated br.ReadVarint(); br.StepIn(dataStart + stringOffset); string str = br.ReadUTF16(); br.StepOut(); strings.Add(str); } FunctionSpecs = new List <FunctionSpec>(); for (int i = 0; i < functionSpecCount; i++) { FunctionSpecs.Add(new FunctionSpec(br, strings)); } Dictionary <long, ConditionDesc> conditions = new Dictionary <long, ConditionDesc>(); for (int i = 0; i < conditionCount; i++) { long offset = br.Position - dataStart; conditions[offset] = new ConditionDesc(br); } CommandSpecs = new List <CommandSpec>(); for (int i = 0; i < commandSpecCount; i++) { CommandSpecs.Add(new CommandSpec(br, strings)); } Dictionary <long, CommandDesc> commands = new Dictionary <long, CommandDesc>(); for (int i = 0; i < commandCount; i++) { long offset = br.Position - dataStart; commands[offset] = new CommandDesc(br, strings); } if (LongFormat) { // Data-start-aligned padding. long offset = br.Position - dataStart; if (offset % 8 > 0) { br.Skip(8 - (int)(offset % 8)); } } Dictionary <long, PassCommandDesc> passCommands = new Dictionary <long, PassCommandDesc>(); for (int i = 0; i < passCommandCount; i++) { long offset = br.Position - dataStart; passCommands[offset] = new PassCommandDesc(br, commands, commandSize); } Dictionary <long, StateDesc> states = new Dictionary <long, StateDesc>(); for (int i = 0; i < stateCount; i++) { long offset = br.Position - dataStart; states[offset] = new StateDesc(br, strings, dataStart, conditions, conditionSize, commands, commandSize, passCommands, passCommandSize); } Machines = new List <MachineDesc>(); for (int i = 0; i < machineCount; i++) { Machines.Add(new MachineDesc(br, strings, states, stateSize)); } if (conditions.Count > 0 || commands.Count > 0 || passCommands.Count > 0 || states.Count > 0) { throw new FormatException("Orphaned ESD descriptions found"); } }
public RnnCell(Executor executor, RnnType rnnType, Variable <T> w, int inputSize, int batch, int hiddenSize, int numLayers, bool isTraining, double dropoutProbability, ulong dropoutSeed = 1337UL) { IsTraining = isTraining; BatchSize = batch; InputSize = inputSize; HiddenSize = hiddenSize; NumLayers = numLayers; RnnType = rnnType; W = w; var context = executor.Context.ToGpuContext(); var dnn = context.Dnn; // state variables var shape = Shape.Create(numLayers, batch, hiddenSize); var strides = Strides.Create(shape[1] * shape[2], shape[2], 1); // inner change most StateDesc.SetND(Dnn.DataTypeOf <T>(), shape.AsInt32Array, strides.AsInt32Array); // xDesc is an array of one element because we do only one step shape = Shape.Create(batch, inputSize, 1); strides = Strides.Create(shape[1] * shape[2], shape[2], 1); var xDesc = new TensorDescriptor(); xDesc.SetND(Dnn.DataTypeOf <T>(), shape.AsInt32Array, strides.AsInt32Array); XDesc = Enumerable.Repeat(xDesc, 1).ToArray(); // yDesc is an array of one element because we do only one step shape = Shape.Create(batch, hiddenSize, 1); strides = Strides.Create(shape[1] * shape[2], shape[2], 1); var yDesc = new TensorDescriptor(); yDesc.SetND(Dnn.DataTypeOf <T>(), shape.AsInt32Array, strides.AsInt32Array); YDesc = Enumerable.Repeat(yDesc, 1).ToArray(); IntPtr dropoutStatesSize; dnn.DropoutGetStatesSize(out dropoutStatesSize); DropoutStates = executor.Context.Device.Allocate <byte>(Shape.Create(dropoutStatesSize.ToInt64())); DropoutDesc.Set(dnn, (float)dropoutProbability, DropoutStates.Buffer.Ptr, dropoutStatesSize, dropoutSeed); var mode = rnnType.Mode; RnnDesc.Set(hiddenSize, numLayers, DropoutDesc, RNNInputMode.LINEAR_INPUT, DirectionMode.UNIDIRECTIONAL, mode, Dnn.DataTypeOf <T>()); IntPtr workSize; dnn.GetRNNWorkspaceSize(RnnDesc, 1, XDesc, out workSize); Workspace = executor.Context.Device.Allocate <byte>(Shape.Create(workSize.ToInt64())); if (isTraining) { IntPtr reserveSize; dnn.GetRNNTrainingReserveSize(RnnDesc, 1, XDesc, out reserveSize); ReserveSize = reserveSize.ToInt64(); //ReserveSpace = executor.AttentionState.Device.Allocate<byte>(Shape.Create(reserveSize.ToInt64())); } IntPtr weightsSize; dnn.GetRNNParamsSize(RnnDesc, xDesc, out weightsSize, Dnn.DataTypeOf <T>()); Util.EnsureTrue(weightsSize.ToInt64() % Gpu.SizeOf <T>() == 0); var shapeW = Shape.Create(weightsSize.ToInt64() / Alea.Gpu.SizeOf <T>()); WDesc.SetND(Dnn.DataTypeOf <T>(), TensorFormat.CUDNN_TENSOR_NCHW, new[] { (int)shapeW[0], 1, 1 }); executor.GetTensor(W, shapeW); if (isTraining) { executor.GetGradient(W, shapeW); } }