void Awake()
{
if (myLifetime > 0f)
{
myClocker = new Clocker();
}
}
public static void TestUserQuery(List <ILanguageModel> models, AccentConverter converter)
{
while (true)
{
Console.Write("Enter a phrase: ");
string data = Console.ReadLine();
if (data.Contains("quit"))
{
break;
}
File.AppendAllText(g_LogFile, data + "\r\n");
Clocker.Tick();
string predictedWords = String.Join(" ", Predict(data, models, converter).Select(wc => wc[0]));
string predicted = String.Format("Predicted: {0} - {1} seconds", predictedWords, Clocker.Seconds());
Console.WriteLine(predicted);
Console.WriteLine();
File.AppendAllText(g_LogFile, predicted + "\r\n\r\n");
}
}
void Awake()
{
if (myLifetime > 0f)
{
myClocker = new Clocker();
}
}
public static void Test(AccentConverter converter, int modelVersion)
{
string modelFile = (converter is VietConverter) ? g_ModelFile : g_FrenchModelFile;
string[] modelFiles = Directory.GetFiles(DataManager.BaseDir, Path.GetFileName(String.Format(modelFile, "*")));
List <ILanguageModel> models = new ILanguageModel[5].ToList();
for (int n = 1; n <= 3; n++)
{
string fileName = String.Format(modelFile, n);
if (!File.Exists(fileName))
{
continue;
}
Console.WriteLine("Loading {0}-gram model...", n);
Clocker.Tick();
models[n - 1] = ModelFactory.LoadModel(modelFiles[n - 1], modelVersion);
Clocker.Tock();
}
TestUserQuery(models, converter);
}
private void MyClock_Loaded(object sender, RoutedEventArgs e)
{
this.DataContext = this;
clockTimer = new Timer((obj) =>
{
this.Dispatcher.BeginInvoke(new Action(() =>
{
Clocker = Clocker.AddSeconds(1);
}), DispatcherPriority.Normal, null);
}, null, 0, 1000);
}
public static void Train(string outModelFilePattern, TupleList <string, string> inputTrainingFiles, int modelVersion, int minGram, int maxGram, AccentConverter converter, bool learnKnownWordsOnly = true)
{
List <int> grams = new List <int>();
for (int n = minGram; n <= maxGram; n++)
{
if (!File.Exists(String.Format(outModelFilePattern, n)))
{
grams.Add(n);
}
}
if (grams.Count == 0)
{
return;
}
// Load dictionary of raw words
Dictionary <string, int> dictionary = learnKnownWordsOnly ? LazyTrainer.ReadDictionary(DataManager.DictionaryFile) : null;
// Load segments from training data
List <List <string> > segments = TextParser.ParseData(inputTrainingFiles);
StringBuilder sbRaw = new StringBuilder();
StringBuilder sbAcc = new StringBuilder();
foreach (int n in grams)
{
int iG = n - 1;
Console.WriteLine("Building {0}-gram ...", iG + 1);
Clocker.Tick();
using (BinaryWriter bwModel = new BinaryWriter(File.Create(String.Format(outModelFilePattern, iG + 1))))
{
ILanguageModel igGram = ModelFactory.CreateModelByVersion(modelVersion);
for (int iS = 0; iS < segments.Count; iS++)
{
List <string> words = segments[iS];
for (int i = 0; i < words.Count - iG; i++)
{
sbRaw.Clear();
sbAcc.Clear();
bool shouldProceed = true;
if (learnKnownWordsOnly)
{
for (int g = 0; g <= iG; g++)
{
string accWord = words[i + g];
string rawWord = converter.Convert(accWord);
if (!dictionary.ContainsKey(rawWord))
{
shouldProceed = false;
break;
}
sbAcc.Append(accWord);
sbRaw.Append(rawWord);
if (g < iG)
{
sbRaw.Append(" ");
}
}
}
else
{
for (int g = 0; g <= iG; g++)
{
sbAcc.Append(words[i + g]);
sbRaw.Append(converter.Convert(words[i + g]));
if (g < iG)
{
sbRaw.Append(" ");
}
}
}
if (shouldProceed)
{
string accents = ExtractAccents(sbAcc.ToString(), converter);
igGram.Add(sbRaw.ToString(), accents);
}
}
}
igGram.WriteToBinary(bwModel);
}
Clocker.Tock();
}
}
private static void TestDataSet(List <string> segments, List <ILanguageModel> models, VietConverter converter, string logFile)
{
Clocker.Tick();
Console.WriteLine("{0} expected total segments", segments.Count);
using (StreamWriter sw = new StreamWriter(File.Create(logFile)))
{
int nCorrectSegments = 0;
int nTotalSegments = 0;
long nCorrectWords = 0;
long nTotalWords = 0;
var logs = new ConcurrentBag <string>();
Parallel.ForEach(
segments,
new ParallelOptions {
MaxDegreeOfParallelism = 2 * Environment.ProcessorCount
},
actualData =>
{
string rawData = converter.Convert(actualData);
string predicted = String.Join(" ", Predict(rawData, models, converter).Select(wc => wc[0]));
string[] wQuery = actualData.Split(new char[0]);
string[] wPredicted = predicted.Split(new char[0]);
bool match = true;
if (wPredicted.Length != wQuery.Length)
{
match = false;
}
else
{
for (int i = 0; i < wQuery.Length; i++)
{
if (wQuery[i] != wPredicted[i])
{
match = false;
wQuery[i] = wQuery[i].ToUpper();
wPredicted[i] = wPredicted[i].ToUpper();
}
else
{
Interlocked.Increment(ref nCorrectWords);
}
}
}
if (!match)
{
logs.Add(String.Format("{0}\r\n{1}\r\n--------------------",
String.Join(" ", wQuery), String.Join(" ", wPredicted)));
}
else
{
Interlocked.Increment(ref nCorrectSegments);
}
Interlocked.Increment(ref nTotalSegments);
if (nTotalSegments % 10000 == 0)
{
Console.WriteLine(nTotalSegments);
}
Interlocked.Add(ref nTotalWords, wQuery.Length);
}
);
foreach (string log in logs)
{
sw.WriteLine(log);
}
Console.WriteLine("{0} total segments, {1}% segment accuracy", nTotalSegments, ((double)nCorrectSegments / nTotalSegments) * 100.0);
Console.WriteLine("{0} total words, {1}% word accuracy", nTotalWords, ((double)nCorrectWords / nTotalWords) * 100.0);
sw.WriteLine("{0} total segments, {1}% segment accuracy", nTotalSegments, ((double)nCorrectSegments / nTotalSegments) * 100.0);
sw.WriteLine("{0} total words, {1}% word accuracy", nTotalWords, ((double)nCorrectWords / nTotalWords) * 100.0);
}
Clocker.Tock();
}
public void Dispose()
{
Background.Dispose();
Canvas.Dispose();
Clocker.Dispose();
}
static void Main(string[] args)
{
// Launch Debugger
Debugger();
Clocker.AddAndStartClock("TotalClock");
Clocker.AddAndStartClock("PipeClock");
if (args.Length != 1)
{
Console.WriteLine("Pipe requires only one argument");
return;
}
PipeClient pipeClient = new PipeClient(args[0]);
pipeClient.Init();
int ySize;
int zSize;
int xSize;
Differ differ;
// Biome, HeightMap, WaterMask and TreeMask arrays/bitmap
Biomes[][] biomes;
int[][] heightMap;
int[][] waterMap;
int[][] treeMap;
float[][] deltaMap;
bool[][] acceptableMap;
int[][] villageMap;
int[][] houseMap;
int[][] roadMap;
int[][] mainRoadMap;
bool[][] lavaMap;
using (BinaryReader reader = pipeClient.ReadMemoryBlock())
{
Material[][][] blocks;
ySize = reader.ReadInt16();
zSize = reader.ReadInt16();
xSize = reader.ReadInt16();
Console.WriteLine($"Y: {ySize} Z: {zSize} X: {xSize}");
blocks = new Material[ySize][][];
for (int y = 0; y < ySize; y++)
{
blocks[y] = new Material[zSize][];
for (int z = 0; z < zSize; z++)
{
blocks[y][z] = new Material[xSize];
for (int x = 0; x < xSize; x++)
{
blocks[y][z][x] = AlphaMaterials.Set.GetMaterial(reader.ReadInt16(), reader.ReadInt16());
}
}
}
differ = new Differ(blocks);
// Biome, HeightMap, WaterMask and TreeMask arrays/bitmap
biomes = new Biomes[zSize][];
heightMap = new int[zSize][];
waterMap = new int[zSize][];
treeMap = new int[zSize][];
deltaMap = new float[zSize][];
acceptableMap = new bool[zSize][];
villageMap = new int[zSize][];
houseMap = new int[zSize][];
roadMap = new int[zSize][];
mainRoadMap = new int[zSize][];
lavaMap = new bool[zSize][];
for (int z = 0; z < zSize; z++)
{
biomes[z] = new Biomes[xSize];
heightMap[z] = new int[xSize];
waterMap[z] = new int[xSize];
treeMap[z] = new int[xSize];
deltaMap[z] = new float[xSize];
acceptableMap[z] = new bool[xSize];
villageMap[z] = new int[xSize];
houseMap[z] = new int[xSize];
roadMap[z] = new int[xSize];
mainRoadMap[z] = new int[xSize];
lavaMap[z] = new bool[xSize];
for (int x = 0; x < xSize; x++)
{
biomes[z][x] = (Biomes)reader.ReadInt16();
heightMap[z][x] = reader.ReadInt16();
waterMap[z][x] = reader.ReadInt16();
treeMap[z][x] = 0;
deltaMap[z][x] = 0;
}
}
}
Clocker.PauseClock("PipeClock");
Console.WriteLine("Ora pro nobis");
Clocker.AddAndStartClock("AlgorithmClock");
{
Tasker.WorkChunk[] workChunks =
{
(int zStart, int zEnd, int xStart, int xEnd) =>
{ HeightMap.FixBoxHeights(differ.World,heightMap, treeMap, zStart, xStart, zEnd, xEnd); }
};
Tasker.Run2DTasks(zSize, xSize, workChunks, null);
}
{
// Delta Map & Lava Map
Tasker.WorkBlock[] workBlocks = { (int z, int x) =>
{
DeltaMap.CalculateDeltaMap(heightMap, waterMap, deltaMap, z, x);
TreeMap.ExpandTreeBlock(z, x, treeMap);
lavaMap[z][x] = LavaMap.isAcceptableLavaMapBlock(heightMap,differ.World, z, x);
} };
Tasker.Run2DTasks(zSize, xSize, null, workBlocks);
}
{
// Acceptable Map
Tasker.WorkBlock[] isAcceptable = { (int z, int x) =>
{
acceptableMap[z][x] = DeltaMap.IsAcceptableBlock(deltaMap, z, x) &&
HeightMap.IsAcceptableTreeMapBlock(treeMap, z, x) &&
waterMap[z][x] != 1 &&
!lavaMap[z][x];
} };
Tasker.Run2DTasks(zSize, xSize, null, isAcceptable);
}
WaterAnalyzer.WaterAnalysis waterAnalysis;
{
int minWaterSize = 20;
waterAnalysis = WaterAnalyzer.AnalyzeWater(waterMap, minWaterSize);
Console.WriteLine($"Min Water Body Size {minWaterSize}");
Console.WriteLine($"Found {waterAnalysis.WaterBodies.Count} valid Water Bodies and {waterAnalysis.InvalidWaterBodies.Count} Invalid ones");
Console.WriteLine("Valid Water Bodies Sizes");
foreach (var waterBody in waterAnalysis.WaterBodies)
{
Console.WriteLine($"\tSize {waterBody.Points.Count}");
}
Console.WriteLine("Invalid Water Bodies Sizes");
foreach (var waterBody in waterAnalysis.InvalidWaterBodies)
{
Console.WriteLine($"\tSize {waterBody.Points.Count}");
}
}
DataQuadTree <Vector2Int> roadQT = new DataQuadTree <Vector2Int>(new Vector2Int(), new Vector2Int(zSize - 1, xSize - 1));
List <VillageMarker> villages;
List <List <Vector2Int> > roads = new List <List <Vector2Int> >();
{
int numberOfTries = 1000;
int expectedVillageSize = 2500;
int radius = 4;
int villageCount = 3;
int maxVillageCount = 3;
villages = VillageDistributor.DistributeVillageMarkers(
acceptableMap, villageMap, waterAnalysis,
villageCount, maxVillageCount, numberOfTries, radius, expectedVillageSize
);
bool mainRoadPlaced = false;
int mainRoadVillageStart = -1, mainRoadVillageEnd = -1;
if (villages.Count > 1)
{
for (int i = 0; i < villages.Count - 1; i++)
{
for (int j = i + 1; j < villages.Count; j++)
{
List <Vector2Int> road = RoadGenerator.FirstRoad(
villages[i].Seed.Z, villages[i].Seed.X,
villages[j].Seed.Z, villages[j].Seed.X,
acceptableMap, deltaMap, waterMap, roadMap, treeMap, houseMap
);
if (road.Count > 0)
{
mainRoadPlaced = true;
mainRoadVillageStart = i;
mainRoadVillageEnd = j;
Console.WriteLine("Main Road length: " + road.Count);
roads.Add(road);
foreach (Vector2Int roadPoint in road)
{
roadQT.Insert(roadPoint, roadPoint);
mainRoadMap[roadPoint.Z][roadPoint.X] = 1;
}
break;
}
}
if (mainRoadPlaced)
{
break;
}
}
}
if (mainRoadPlaced)
{
for (int i = 0; i < villages.Count; ++i)
{
if (i != mainRoadVillageStart && i != mainRoadVillageEnd)
{
Console.WriteLine($"Connecting village: {i} to roads");
List <Vector2Int> road = RoadGenerator.PointToRoad(
villages[i].Seed.Z, villages[i].Seed.X,
acceptableMap, deltaMap, waterMap, roadMap, treeMap, houseMap,
roadQT
);
roads.Add(road);
foreach (Vector2Int roadPoint in road)
{
roadQT.Insert(roadPoint, roadPoint);
}
}
}
//RoadPlacer.RoadsPlacer(roads, roadMap, heightMap, waterMap, biomes, differ);
}
else
{
Console.WriteLine("Failed to Place all the roads");
}
}
DataQuadTree <RectInt> villagesQT = new DataQuadTree <RectInt>(new Vector2Int(0, 0), new Vector2Int(zSize, xSize));
foreach (VillageMarker village in villages)
{
HouseDistributor.FillVillage(deltaMap, heightMap, acceptableMap, houseMap, roadMap, villageMap,
waterMap, treeMap, biomes, village, differ.World, new Vector2Int(8, 8), differ,
villagesQT, roadQT, ref roads);
}
RoadPlacer.RoadsPlacer(roads, roadMap, heightMap, waterMap, biomes, differ);
{
// Drawing
Mapper.SaveMapInfo[] saveMapInfos =
{
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "villagemap",
colorWork = (int z, int x) =>{
if (villageMap[z][x] >= 1)
{
return(Color.Yellow);
}
else if (acceptableMap[z][x] && villageMap[z][x] <= 0)
{
return(Color.Orange);
}
return(Color.Transparent);
},
specialColors = (Mapper.ColorApplier colorApplier) =>
{
for (int i = 0; i < villages.Count; i++)
{
colorApplier.Invoke(villages[i].Seed.Z, villages[i].Seed.X, Color.Blue);
}
}
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "acceptablemap",
colorWork = (int z, int x) =>{
if (acceptableMap[z][x])
{
return(Color.Green);
}
else
{
return(Color.Red);
}
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "deltamap",
colorWork = (int z, int x) =>{
if (0 <= deltaMap[z][x] && deltaMap[z][x] <= DeltaMap.kMaxDelta)
{
float tVal = 1.0f - deltaMap[z][x] / DeltaMap.kMaxDelta;
return(Color.FromRgba(0, (byte)(255.0f * tVal + 100.0f * (1.0f - tVal)), 0, 255));
}
else if (deltaMap[z][x] > DeltaMap.kMaxDelta)
{
return(Color.Red);
}
return(Color.Transparent);
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "treemap",
colorWork = (int z, int x) =>{
if (treeMap[z][x] == 1)
{
return(Color.Green);
}
else if (treeMap[z][x] == 2)
{
return(Color.Brown);
}
else if (treeMap[z][x] == 3)
{
return(Color.DarkSeaGreen);
}
return(Color.Transparent);
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "heightmap",
colorWork = (int z, int x) =>{
if (heightMap[z][x] >= 0)
{
return(Color.FromRgba((byte)heightMap[z][x], (byte)heightMap[z][x], (byte)heightMap[z][x], 255));
}
else
{
return(Color.Red);
}
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "watermap",
colorWork = (int z, int x) =>{
if (waterMap[z][x] == 1)
{
return(Color.Blue);
}
return(Color.Transparent);
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "houseMap",
colorWork = (int z, int x) =>{
if (houseMap[z][x] == 1)
{
return(Color.Brown);
}
return(Color.Transparent);
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "roadMap",
colorWork = (int z, int x) =>{
switch (roadMap[z][x])
{
case RoadGenerator.MainRoadMarker:
return(Color.PaleVioletRed);
case RoadGenerator.RoadMarker:
return(Color.Purple);
case RoadGenerator.BridgeMarker:
return(Color.Brown);
case RoadGenerator.MainBridgeMarker:
return(Color.BurlyWood);
default:
return(Color.Transparent);
}
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "mainRoadMap",
colorWork = (int z, int x) =>{
if (mainRoadMap[z][x] == 1)
{
return(Color.Turquoise);
}
return(Color.Transparent);
},
specialColors = null
},
new Mapper.SaveMapInfo {
zSize = zSize, xSize = xSize, name = "lavaMap",
colorWork = (int z, int x) =>{
if (lavaMap[z][x])
{
return(Color.DarkRed);
}
return(Color.Transparent);
},
specialColors = null
}
};
Mapper.SaveMaps(saveMapInfos);
}
// Write Data Back to Python
{
using (BinaryWriter writer = new BinaryWriter(new MemoryStream()))
{
differ.SerializeChanges(writer);
// Return data To Python
Console.WriteLine(writer.BaseStream.Length);
pipeClient.WriteMemoryBlock((MemoryStream)writer.BaseStream);
}
}
// Close Pipe
pipeClient.DeInit();
Clocker.PauseClock("AlgorithmClock");
Clocker.PauseClock("TotalClock");
double pipeTime = Clocker.GetTime("PipeClock", true);
double algoTime = Clocker.GetTime("AlgorithmClock", true);
double totalTime = Clocker.GetTime("TotalClock", true);
Clocker.PrintAllTime(true, true, true);
Clocker.RemoveClock("PipeClock", true, true, true);
Clocker.RemoveClock("AlgorithmClock", true, true, true);
Clocker.RemoveClock("TotalClock", true, true, true);
Console.WriteLine($"Pipe Percentage of Time: {pipeTime / totalTime * 100.0}% ({pipeTime} secs)");
Console.WriteLine($"Algorithm Percentage of Time: {algoTime / totalTime * 100.0}% ({algoTime} secs)");
}
