/// <exception cref="System.Exception"/>
public virtual void TestMlock()
{
Assume.AssumeTrue(NativeIO.IsAvailable());
FilePath TestFile = new FilePath(new FilePath(Runtime.GetProperty("test.build.data"
, "build/test/data")), "testMlockFile");
int BufLen = 12289;
byte[] buf = new byte[BufLen];
int bufSum = 0;
for (int i = 0; i < buf.Length; i++)
{
buf[i] = unchecked ((byte)(i % 60));
bufSum += buf[i];
}
FileOutputStream fos = new FileOutputStream(TestFile);
try
{
fos.Write(buf);
fos.GetChannel().Force(true);
}
finally
{
fos.Close();
}
FileInputStream fis = null;
FileChannel channel = null;
try
{
// Map file into memory
fis = new FileInputStream(TestFile);
channel = fis.GetChannel();
long fileSize = channel.Size();
MappedByteBuffer mapbuf = channel.Map(FileChannel.MapMode.ReadOnly, 0, fileSize);
// mlock the buffer
NativeIO.POSIX.Mlock(mapbuf, fileSize);
// Read the buffer
int sum = 0;
for (int i_1 = 0; i_1 < fileSize; i_1++)
{
sum += mapbuf.Get(i_1);
}
Assert.Equal("Expected sums to be equal", bufSum, sum);
// munmap the buffer, which also implicitly unlocks it
NativeIO.POSIX.Munmap(mapbuf);
}
finally
{
if (channel != null)
{
channel.Close();
}
if (fis != null)
{
fis.Close();
}
}
}
Load(long length, FileInputStream blockIn, FileInputStream metaIn, string blockFileName
)
{
Org.Apache.Hadoop.Hdfs.Server.Datanode.Fsdataset.Impl.MappableBlock mappableBlock
= null;
MappedByteBuffer mmap = null;
FileChannel blockChannel = null;
try
{
blockChannel = blockIn.GetChannel();
if (blockChannel == null)
{
throw new IOException("Block InputStream has no FileChannel.");
}
mmap = blockChannel.Map(FileChannel.MapMode.ReadOnly, 0, length);
NativeIO.POSIX.GetCacheManipulator().Mlock(blockFileName, mmap, length);
VerifyChecksum(length, metaIn, blockChannel, blockFileName);
mappableBlock = new Org.Apache.Hadoop.Hdfs.Server.Datanode.Fsdataset.Impl.MappableBlock
(mmap, length);
}
finally
{
IOUtils.CloseQuietly(blockChannel);
if (mappableBlock == null)
{
if (mmap != null)
{
NativeIO.POSIX.Munmap(mmap);
}
}
}
// unmapping also unlocks
return(mappableBlock);
}
private MappedByteBuffer getModel(AssetManager assets, string modelFilename)
{
AssetFileDescriptor fileDescriptor = assets.OpenFd(modelFilename);
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
return(fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength));
}
/*
* Memory-map the model file in Assets.
*/
public static MappedByteBuffer LoadModelFile(Context context, string modelFile)
{
AssetFileDescriptor fileDescriptor = context.Assets.OpenFd(modelFile);
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
return(fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength));
}
private void LoadTensorflowModel()
{
var assets = Android.App.Application.Context.Assets;
AssetFileDescriptor fileDescriptor = assets.OpenFd(MODEL_FILE);
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
_model = fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
Interpreter.Options options = new Interpreter.Options();
options.SetNumThreads(THREAD_NUM);
_interpreter = new Interpreter(_model, options);
}
internal virtual MappedByteBuffer LoadMmapInternal()
{
try
{
FileChannel channel = dataStream.GetChannel();
MappedByteBuffer mmap = channel.Map(FileChannel.MapMode.ReadOnly, 0, Math.Min(int.MaxValue
, channel.Size()));
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": created mmap of size " + channel.Size());
}
return(mmap);
}
catch (IOException e)
{
Log.Warn(this + ": mmap error", e);
return(null);
}
catch (RuntimeException e)
{
Log.Warn(this + ": mmap error", e);
return(null);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestCopyFileUnbuffered()
{
string MethodName = GenericTestUtils.GetMethodName();
FilePath srcFile = new FilePath(TestDir, MethodName + ".src.dat");
FilePath dstFile = new FilePath(TestDir, MethodName + ".dst.dat");
int fileSize = unchecked ((int)(0x8000000));
// 128 MB
int Seed = unchecked ((int)(0xBEEF));
int batchSize = 4096;
int numBatches = fileSize / batchSize;
Random rb = new Random(Seed);
FileChannel channel = null;
RandomAccessFile raSrcFile = null;
try
{
raSrcFile = new RandomAccessFile(srcFile, "rw");
channel = raSrcFile.GetChannel();
byte[] bytesToWrite = new byte[batchSize];
MappedByteBuffer mapBuf;
mapBuf = channel.Map(FileChannel.MapMode.ReadWrite, 0, fileSize);
for (int i = 0; i < numBatches; i++)
{
rb.NextBytes(bytesToWrite);
mapBuf.Put(bytesToWrite);
}
NativeIO.CopyFileUnbuffered(srcFile, dstFile);
Assert.Equal(srcFile.Length(), dstFile.Length());
}
finally
{
IOUtils.Cleanup(Log, channel);
IOUtils.Cleanup(Log, raSrcFile);
FileUtils.DeleteQuietly(TestDir);
}
}
public List <string> PredictNextWord(string word)
{
switch (Language)
{
case "pt":
using (Stream ms = context.Assets.Open("Models/idx2word_pt.txt"))
{
using (var sr = new StreamReader(ms))
{
idx2word = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, String> >(sr.ReadToEnd());
}
}
using (Stream ms = context.Assets.Open("Models/word2idx_pt.txt"))
{
using (var sr = new StreamReader(ms))
{
word2idx = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <String, int> >(sr.ReadToEnd());
}
}
if (!word2idx.ContainsKey(word))
{
return(new List <string>());
}
var il = new float[10];
for (var i = 0; i < 10; i++)
{
il[i] = (float)(word2idx[word]);
}
if (model == null)
{
var assets = Application.Context.Assets;
AssetFileDescriptor fileDescriptor = assets.OpenFd("Models/predictor_pt.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
var asd = fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
model = new Xamarin.TensorFlow.Lite.Interpreter(asd);
}
byte[] ibytes = new byte[il.Length * sizeof(float)];
Buffer.BlockCopy(il, 0, ibytes, 0, ibytes.Length);
var bytebuffer = Java.Nio.ByteBuffer.Wrap(ibytes);
var output = Java.Nio.ByteBuffer.AllocateDirect(4 * 10 * (idx2word.Count + 1));
model.Run(bytebuffer, output);
List <Double> res = new List <double>();
var buffer = new byte[4 * 10 * (idx2word.Count + 1)];
Marshal.Copy(output.GetDirectBufferAddress(), buffer, 0, 4 * 10 * (idx2word.Count + 1));
for (var i = 0; i < idx2word.Count + 1; i++)
{
res.Add(BitConverter.ToSingle(buffer, i * 4));
}
return(res.OrderByDescending(x => x).Select(x2 => idx2word[res.IndexOf(x2)]).Take(10).ToList());
case "es":
using (Stream ms = context.Assets.Open("Models/idx2word_es.txt"))
{
using (var sr = new StreamReader(ms))
{
idx2word = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, String> >(sr.ReadToEnd());
}
}
using (Stream ms = context.Assets.Open("Models/word2idx_es.txt"))
{
using (var sr = new StreamReader(ms))
{
word2idx = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <String, int> >(sr.ReadToEnd());
}
}
if (!word2idx.ContainsKey(word))
{
return(new List <string>());
}
var il2 = new float[10];
for (var i = 0; i < 10; i++)
{
il2[i] = (float)(word2idx[word]);
}
if (model == null)
{
var assets = Application.Context.Assets;
AssetFileDescriptor fileDescriptor = assets.OpenFd("Models/predictor_es.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
var asd = fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
model = new Xamarin.TensorFlow.Lite.Interpreter(asd);
}
byte[] ibytes2 = new byte[il2.Length * sizeof(float)];
Buffer.BlockCopy(il2, 0, ibytes2, 0, ibytes2.Length);
var bytebuffer2 = Java.Nio.ByteBuffer.Wrap(ibytes2);
var output2 = Java.Nio.ByteBuffer.AllocateDirect(4 * 10 * (idx2word.Count + 1));
model.Run(bytebuffer2, output2);
List <Double> res2 = new List <double>();
var buffer2 = new byte[4 * 10 * (idx2word.Count + 1)];
Marshal.Copy(output2.GetDirectBufferAddress(), buffer2, 0, 4 * 10 * (idx2word.Count + 1));
for (var i = 0; i < idx2word.Count + 1; i++)
{
res2.Add(BitConverter.ToSingle(buffer2, i * 4));
}
return(res2.OrderByDescending(x => x).Select(x2 => idx2word[res2.IndexOf(x2)]).Take(10).ToList());
default:
using (Stream ms = context.Assets.Open("Models/idx2word_en.txt"))
{
using (var sr = new StreamReader(ms))
{
idx2word = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <int, String> >(sr.ReadToEnd());
}
}
using (Stream ms = context.Assets.Open("Models/word2idx_en.txt"))
{
using (var sr = new StreamReader(ms))
{
word2idx = Newtonsoft.Json.JsonConvert.DeserializeObject <Dictionary <String, int> >(sr.ReadToEnd());
}
}
if (!word2idx.ContainsKey(word))
{
return(new List <string>());
}
var il3 = new float[10];
for (var i = 0; i < 10; i++)
{
il3[i] = (float)(word2idx[word]);
System.Diagnostics.Debug.WriteLine(il3[i]);
}
if (model == null)
{
var assets = Application.Context.Assets;
AssetFileDescriptor fileDescriptor = assets.OpenFd("Models/predictor_en.tflite");
FileInputStream inputStream = new FileInputStream(fileDescriptor.FileDescriptor);
FileChannel fileChannel = inputStream.Channel;
long startOffset = fileDescriptor.StartOffset;
long declaredLength = fileDescriptor.DeclaredLength;
var asd = fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
model = new Xamarin.TensorFlow.Lite.Interpreter(asd);
}
byte[] ibytes3 = new byte[il3.Length * sizeof(float)];
Buffer.BlockCopy(il3, 0, ibytes3, 0, ibytes3.Length);
var bytebuffer3 = Java.Nio.ByteBuffer.Wrap(ibytes3);
var output3 = Java.Nio.ByteBuffer.AllocateDirect(4 * 10 * (idx2word.Count + 1));
model.Run(bytebuffer3, output3);
List <Double> res3 = new List <double>();
var buffer3 = new byte[4 * 10 * (idx2word.Count + 1)];
Marshal.Copy(output3.GetDirectBufferAddress(), buffer3, 0, 4 * 10 * (idx2word.Count + 1));
for (var i = 0; i < idx2word.Count + 1; i++)
{
res3.Add(BitConverter.ToSingle(buffer3, i * 4));
}
return(res3.OrderByDescending(x => x).Select(x2 => idx2word[res3.IndexOf(x2)]).Take(10).ToList());
}
}
