/// <summary>
/// インデックスなしでの検索です。
/// </summary>
/// <param name="csvFilePath"></param>
/// <param name="keyword"></param>
/// <returns></returns>
private static IEnumerable <string> SearchKeywordNoIndex(string csvFilePath, string keyword)
{
var splitKeywords = IndexUtility.SplitKeyword(keyword, 2).ToList();
using (var reader = new CsvReader(new StreamReader(csvFilePath, Encoding.GetEncoding("Shift-JIS"))))
{
while (reader.Read())
{
var containCheckList = new bool[splitKeywords.Count()];
foreach (var cellData in reader.Context.Record)
{
foreach (var item in splitKeywords.Select((value, index) => new { value, index }))
{
if (cellData.Contains(item.value))
{
containCheckList[item.index] = true;
}
}
}
if (containCheckList.All(isContain => isContain))
{
string text = string.Join(",", reader.Context.Record);
yield return(text);
}
}
}
}
public DateTime GetDateOfProviderList()
{
var index = _elasticsearchCustomClient.GetIndicesPointingToAlias(_applicationSettings.RoatpProviderIndexAlias).FirstOrDefault();
_log.Info($"Trying to get date for index: {index} with alias: {_applicationSettings.RoatpProviderIndexAlias}");
return(IndexUtility.GetDateFromIndexNameAndDateExtension(index, _applicationSettings.RoatpProviderIndexAlias));
}
public void SaveAndLoadIndexData()
{
var expect = new Dictionary <string, List <int> >
{
{ "0", new List <int> {
1, 2, 4
} },
{ "東京", new List <int> {
2, 5, 9
} },
{ "大阪", new List <int> {
4, 8
} }
};
string saveFilePath =
Path.Combine(TestSetup.RootDirectory, "TestResult", "Utilties", "IndexUtility", "SaveIndexData",
"test.idx");
var encoding = Encoding.GetEncoding("Shift-JIS");
IndexUtility.SaveIndexData(saveFilePath, expect, encoding);
var actual = IndexUtility.LoadIndexData(saveFilePath, encoding);
CollectionAssert.AreEquivalent(expect, actual);
}
/// <summary>
/// レコードを検索します。
/// </summary>
/// <param name="keyword">検索キーワード</param>
public IEnumerable <string> SearchRecord(string keyword)
{
string filePath = FileUtility.GetIndexFilePath(_storageDirectoryPath);
if (String.IsNullOrEmpty(filePath))
{
Console.WriteLine("インデックスがありません。先にインデックスを作成してください。");
yield break;
}
var keywordSplit = IndexUtility.SplitKeyword(keyword, Ngram).ToList();
var indices = IndexUtility.LoadIndexData(filePath, FileEncoding, keywordSplit);
var result = IndexUtility.SearchKeywordLines(keywordSplit.FirstOrDefault(), indices);
foreach (var item in keywordSplit.Skip(1))
{
var next = IndexUtility.SearchKeywordLines(item, indices).ToList();
result = result.Intersect(next);
}
var resultList = result.ToList();
string csvFilePath = FileUtility.GetCsvFilePath(_storageDirectoryPath);
foreach (var text in FileUtility.GetFileLines(resultList, csvFilePath, FileEncoding))
{
yield return(text);
}
}
/// <summary>
/// インデックスの作成を行う処理です。
/// </summary>
public void CreateIndex()
{
string zipFilePath = Path.Combine(_storageDirectoryPath, "download.zip");
FileUtility.DownloadZipFile(zipFilePath, DataSourceUrl);
string extractDirectory = FileUtility.GetZipExtractDirectory(_storageDirectoryPath);
string csvFilePath = FileUtility.ExtractCsvFile(zipFilePath, extractDirectory);
var indexData = IndexUtility.GetIndexData(csvFilePath, Ngram, FileEncoding);
string indexFilePath = csvFilePath + FileUtility.IndexFileExtension;
IndexUtility.SaveIndexData(indexFilePath, indexData, FileEncoding);
}
public static int3 FindInLayer(DynamicBuffer <AABB> bounds, int3 dimensions, float3 target, int3 begin, int3 end)
{
for (int z = begin.z; z < end.z; z++)
{
for (int y = begin.y; y < end.y; y++)
{
for (int x = begin.x; x < end.x; x++)
{
var coords = new int3(x, y, z);
if (bounds[IndexUtility.CoordsToIdx(coords, dimensions)].Contains(target))
{
return(coords);
}
}
}
}
return(-1);
}
public void LoadIndexDataArgumentKeywords()
{
var indices = new Dictionary <string, List <int> >
{
{ "0", new List <int> {
1, 2, 4
} },
{ "関東", new List <int> {
2, 5, 9
} },
{ "大阪", new List <int> {
4, 8
} },
{ "京都", new List <int> {
5, 9
} }
};
var expect = new Dictionary <string, List <int> >();
var keywords = new List <string> {
"東", "京"
};
foreach (var index in indices)
{
if (keywords.Any(keyword => index.Key.Contains(keyword)))
{
expect.Add(index.Key, index.Value);
}
}
Assert.IsTrue(Directory.Exists(Path.Combine(TestSetup.RootDirectory, "TestResult", "Utilties")));
string saveFilePath =
Path.Combine(TestSetup.RootDirectory, "TestResult", "Utilties", "IndexUtility", "SaveIndexData",
"test.idx");
var encoding = Encoding.GetEncoding("Shift-JIS");
IndexUtility.SaveIndexData(saveFilePath, indices, encoding);
var actual = IndexUtility.LoadIndexData(saveFilePath, encoding, keywords);
CollectionAssert.AreEquivalent(expect, actual);
}
/// <summary>
/// 各パーツメッシュの持つインデックスをすべて結合し、ひとつの配列にして返す。
/// その際各メッシュの頂点数は、次のインデックスのベースとなる。
/// また、マテリアル別のサブメッシュも、ひとつに統合される。
/// </summary>
public static int[][] ToIndicesArray
(
IEnumerable <Vector3[]> vertices_PerMesh,
IEnumerable <IEnumerable <int[]> > indices_PerSubmeshPerMesh,
IEnumerable <Matrix4x4> mtPart_PerMesh
)
{
var mts = mtPart_PerMesh;
var qBaseVtxs = PerMesh.QueryBaseVertex(vertices_PerMesh);
var qIndex =
from xyz_ in (indices_PerSubmeshPerMesh, mts, qBaseVtxs).Zip()
select(idxss: xyz_.x, mt: xyz_.y, baseVtx: xyz_.z) into srcPerMesh
from idxs in srcPerMesh.idxss
from index in IndexUtility.ReverseEvery3_IfMinusScale(idxs, srcPerMesh.mt)
select srcPerMesh.baseVtx + index;
return(new int[][] { qIndex.ToArray() });
//return Enumerable.Repeat( qIndex, 1 ).ToArrayRecursive2();
}
/// <summary>
/// Enqueues the specified values for writing.
/// </summary>
/// <param name="tag">The tag that the values are being written for.</param>
/// <param name="values">The values to write to the archive.</param>
/// <param name="archiveCandidate">The current archive candidate value for the tag.</param>
internal void WriteValues(ElasticsearchTagDefinition tag, IEnumerable <TagValue> values, ArchiveCandidateValue archiveCandidate)
{
_valuesLock.EnterReadLock();
try {
foreach (var value in values ?? new TagValue[0])
{
var op = new BulkIndexOperation <TagValueDocument>(value.ToTagValueDocument(tag, null))
{
Index = IndexUtility.GetIndexNameForArchiveTagValue(_historian.ArchiveIndexNamePrefix, tag, value.UtcSampleTime, _historian.ArchiveIndexSuffixGenerator)
};
_nextInsert.AddOperation(op);
}
if (archiveCandidate != null)
{
_archiveCandidateValues[tag.IdAsGuid] = archiveCandidate;
}
}
finally {
_valuesLock.ExitReadLock();
}
}
unsafe protected override void OnUpdate()
{
if (mSystemsBusy != 0)
{
return;
}
UnityEngine.Profiling.Profiler.BeginSample("Finding Player");
// Gather Necessary Entities
float3 playerPos;
{
var tmp = mPlayerQuery.ToComponentDataArray <Unity.Transforms.Translation>(Allocator.TempJob);
playerPos = tmp[0].Value;
tmp.Dispose();
}
Entity grid;
{
var tmp = mVoxelQuery.ToEntityArray(Allocator.TempJob);
grid = tmp[0];
tmp.Dispose();
}
var layers = EntityManager.GetBuffer <Layers>(grid).Reinterpret <Entity>();
var dimensions = EntityManager.GetBuffer <Dimensions>(grid).Reinterpret <int3>();
UnityEngine.Profiling.Profiler.BeginSample("Find Player In Layer");
// Find Player in Layer
// TODO: Check if player is within same bounds as last time, cause In that case we don't need to do anything.
var targetCoords = int3.zero;
{
var prevDim = new int3(1);
for (int i = 1; i < layers.Length; i++)
{
var lengths = dimensions[i] / prevDim;
var begin = IndexUtility.CoordsToSubGrid(targetCoords, prevDim, dimensions[i]);
var end = begin + lengths;
var bounds = EntityManager.GetBuffer <VoxelAABB>(layers[i]).Reinterpret <AABB>();
targetCoords = FindInLayer(bounds, dimensions[i], playerPos, begin, end);
prevDim = dimensions[i];
if (targetCoords.x == -1)
{
break;
}
}
}
UnityEngine.Profiling.Profiler.EndSample();
UnityEngine.Profiling.Profiler.BeginSample("Find Relevant Bounds");
// Find All Relevant Bounds
mRelevantBounds.Clear();
// We couldn't find the player in any of the bounds.
// We don't just want to return here, but we want to skip the step of finding relevant bounds.
if (targetCoords.x >= 0)
{
const int boxesInEachDirection = 3;
var lowestLayer = dimensions[dimensions.Length - 1];
for (int z = math.max(targetCoords.z - boxesInEachDirection, 0); z <= math.min(targetCoords.z + boxesInEachDirection, lowestLayer.z - 1); z++)
{
for (int y = math.max(targetCoords.y - boxesInEachDirection, 0); y <= math.min(targetCoords.y + boxesInEachDirection, lowestLayer.y - 1); y++)
{
for (int x = math.max(targetCoords.x - boxesInEachDirection, 0); x <= math.min(targetCoords.x + boxesInEachDirection, lowestLayer.x - 1); x++)
{
// "Me"
var coords = new int3(x, y, z);
mRelevantBounds.TryAdd(new Voxel {
Layer = layers.Length - 1, Index = IndexUtility.CoordsToIdx(coords, lowestLayer)
}, new Empty());
// Everyone who contains "Me", Ignoring top layer box as that is static.
for (int i = layers.Length - 2; i > 0; i--)
{
mRelevantBounds.TryAdd(new Voxel {
Layer = i, Index = IndexUtility.CoordsToIdx(IndexUtility.CoordsToOuterGrid(coords, lowestLayer, dimensions[i]), dimensions[i])
}, new Empty());
}
}
}
}
}
UnityEngine.Profiling.Profiler.EndSample();
UnityEngine.Profiling.Profiler.BeginSample("Mark Bounds for Deletion");
//Deletion
// Mark Entities for Deletion
Entities.With(mCurrentVoxelsQuery).ForEach((Entity e, ref StreamedInVoxel v) =>
{
if (!mRelevantBounds.TryGetValue(v.Value, out Empty _))
{
PostUpdateCommands.AddComponent(e, new VoxelRemovalTag());
}
else
{
mRelevantBounds.Remove(v.Value);
}
});
var toStream = mRelevantBounds.GetKeyArray(Allocator.Temp);
for (int i = 0; i < toStream.Length; i++)
{
PostUpdateCommands.SetComponent(PostUpdateCommands.CreateEntity(mRequestArchetype), new StreamedInVoxel {
Value = toStream[i]
});
}
toStream.Dispose();
UnityEngine.Profiling.Profiler.EndSample();
UnityEngine.Profiling.Profiler.EndSample();
}
public void SplitNgram(string text, int ngram, List <string> expect)
{
var result = IndexUtility.SplitNgram(text, ngram);
CollectionAssert.AreEqual(expect, result);
}
public IList <string> SplitKeyword(string keyword, int ngram)
{
return(IndexUtility.SplitKeyword(keyword, ngram).ToList());
}
public void SearchKeywordLines(string keyword, Dictionary <string, List <int> > indices, List <int> expected)
{
var actual = IndexUtility.SearchKeywordLines(keyword, indices).ToList();
CollectionAssert.AreEqual(expected, actual);
}
