// noop, expecting an exception
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureCloseKeyStreamManyTimesInWriter()
{
if (skip)
{
return;
}
DataOutputStream outKey = writer.PrepareAppendKey(4);
try
{
outKey.Write(Runtime.GetBytesForString("key0"));
outKey.Close();
}
catch (Exception)
{
}
finally
{
// noop, expecting an exception
try
{
outKey.Close();
}
catch (Exception)
{
}
}
// no-op
outKey.Close();
outKey.Close();
Assert.True("Multiple close should have no effect.", true);
}
/// <exception cref="System.Exception"/>
private static void UploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi
, Configuration configuration, IList <string> rootLogDirs, NodeId nodeId, ContainerId
containerId, Path appDir, FileSystem fs)
{
Path path = new Path(appDir, LogAggregationUtils.GetNodeString(nodeId) + Runtime.
CurrentTimeMillis());
AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration
, path, ugi);
writer.WriteApplicationOwner(ugi.GetUserName());
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName();
writer.WriteApplicationACLs(appAcls);
DataOutputStream @out = writer.GetWriter().PrepareAppendKey(-1);
new AggregatedLogFormat.LogKey(containerId).Write(@out);
@out.Close();
@out = writer.GetWriter().PrepareAppendValue(-1);
new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser
().GetShortUserName()).Write(@out, new HashSet <FilePath>());
@out.Close();
writer.Close();
}
// noop, expecting an exception
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailureValueTooLong()
{
if (skip)
{
return;
}
DataOutputStream outKey = writer.PrepareAppendKey(4);
outKey.Write(Runtime.GetBytesForString("key0"));
outKey.Close();
DataOutputStream outValue = writer.PrepareAppendValue(3);
try
{
outValue.Write(Runtime.GetBytesForString("value0"));
outValue.Close();
NUnit.Framework.Assert.Fail("Value is longer than expected.");
}
catch (Exception)
{
}
// noop, expecting an exception
try
{
outKey.Close();
outKey.Close();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Second or more close() should have no effect.");
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void WriteApplicationOwner(string user)
{
DataOutputStream @out = this.writer.PrepareAppendKey(-1);
ApplicationOwnerKey.Write(@out);
@out.Close();
@out = this.writer.PrepareAppendValue(-1);
@out.WriteUTF(user);
@out.Close();
}
/// <exception cref="System.IO.IOException"/>
private void WriteVersion()
{
DataOutputStream @out = this.writer.PrepareAppendKey(-1);
VersionKey.Write(@out);
@out.Close();
@out = this.writer.PrepareAppendValue(-1);
@out.WriteInt(Version);
@out.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual void WriteApplicationACLs(IDictionary <ApplicationAccessType, string
> appAcls)
{
DataOutputStream @out = this.writer.PrepareAppendKey(-1);
ApplicationAclKey.Write(@out);
@out.Close();
@out = this.writer.PrepareAppendValue(-1);
foreach (KeyValuePair <ApplicationAccessType, string> entry in appAcls)
{
@out.WriteUTF(entry.Key.ToString());
@out.WriteUTF(entry.Value);
}
@out.Close();
}
/// <exception cref="System.IO.IOException"/>
private int WritePrepWithUnkownLength(TFile.Writer writer, int start, int n)
{
for (int i = start; i < (start + n); i++)
{
DataOutputStream @out = writer.PrepareAppendKey(-1);
string localKey = string.Format(localFormatter, i);
@out.Write(Runtime.GetBytesForString(localKey));
@out.Close();
string value = "value" + localKey;
@out = writer.PrepareAppendValue(-1);
@out.Write(Runtime.GetBytesForString(value));
@out.Close();
}
return(start + n);
}
public virtual void TestFailureGetNonExistentMetaBlock()
{
if (skip)
{
return;
}
writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString
("valueX"));
// create a new metablock
DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm
.Gz.GetName());
outMeta.Write(123);
outMeta.Write(Runtime.GetBytesForString("foo"));
outMeta.Close();
CloseOutput();
TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen
(), conf);
DataInputStream mb = reader.GetMetaBlock("testX");
NUnit.Framework.Assert.IsNotNull(mb);
mb.Close();
try
{
DataInputStream mbBad = reader.GetMetaBlock("testY");
NUnit.Framework.Assert.Fail("Error on handling non-existent metablocks.");
}
catch (Exception)
{
}
// noop, expecting exceptions
reader.Close();
}
private static void CreateBinaryTokenFile(Configuration conf)
{
// Fetch delegation tokens and store in binary token file.
try
{
Credentials cred1 = new Credentials();
Credentials cred2 = new Credentials();
TokenCache.ObtainTokensForNamenodesInternal(cred1, new Path[] { p1 }, conf);
foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t in cred1.GetAllTokens
())
{
cred2.AddToken(new Text(DelegationTokenKey), t);
}
DataOutputStream os = new DataOutputStream(new FileOutputStream(binaryTokenFileName
.ToString()));
try
{
cred2.WriteTokenStorageToStream(os);
}
finally
{
os.Close();
}
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail("Exception " + e);
}
}
public virtual void TestFailureWriteMetaBlocksWithSameName()
{
if (skip)
{
return;
}
writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString
("valueX"));
// create a new metablock
DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm
.Gz.GetName());
outMeta.Write(123);
outMeta.Write(Runtime.GetBytesForString("foo"));
outMeta.Close();
// add the same metablock
try
{
writer.PrepareMetaBlock("testX", Compression.Algorithm.Gz.GetName());
NUnit.Framework.Assert.Fail("Cannot create metablocks with the same name.");
}
catch (Exception)
{
}
// noop, expecting exceptions
CloseOutput();
}
public virtual void TestLocalFileCorruption()
{
Configuration conf = new HdfsConfiguration();
Path file = new Path(PathUtils.GetTestDirName(GetType()), "corruptFile");
FileSystem fs = FileSystem.GetLocal(conf);
DataOutputStream dos = fs.Create(file);
dos.WriteBytes("original bytes");
dos.Close();
// Now deliberately corrupt the file
dos = new DataOutputStream(new FileOutputStream(file.ToString()));
dos.WriteBytes("corruption");
dos.Close();
// Now attempt to read the file
DataInputStream dis = fs.Open(file, 512);
try
{
System.Console.Out.WriteLine("A ChecksumException is expected to be logged.");
dis.ReadByte();
}
catch (ChecksumException)
{
}
//expect this exception but let any NPE get thrown
fs.Delete(file, true);
}
/// <exception cref="System.IO.IOException"/>
public override void StoreTokenMasterKey(DelegationKey key)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing master key " + key.GetKeyId());
}
Path keyPath = new Path(tokenKeysStatePath, TokenMasterKeyFilePrefix + key.GetKeyId
());
if (fs.Exists(keyPath))
{
throw new IOException(keyPath + " already exists");
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
key.Write(dataStream);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
CreateNewFile(keyPath, memStream.ToByteArray());
}
public virtual void TestStatusLimit()
{
Path test = new Path(testRootTempDir, "testStatusLimit");
Configuration conf = new Configuration();
Path inDir = new Path(test, "in");
Path outDir = new Path(test, "out");
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(inDir))
{
fs.Delete(inDir, true);
}
fs.Mkdirs(inDir);
DataOutputStream file = fs.Create(new Path(inDir, "part-" + 0));
file.WriteBytes("testStatusLimit");
file.Close();
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
Job job = Job.GetInstance(conf, "testStatusLimit");
job.SetMapperClass(typeof(TestReporter.StatusLimitMapper));
job.SetNumReduceTasks(0);
FileInputFormat.AddInputPath(job, inDir);
FileOutputFormat.SetOutputPath(job, outDir);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
public virtual void TestLease()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
Path a = new Path(dir, "a");
Path b = new Path(dir, "b");
DataOutputStream a_out = fs.Create(a);
a_out.WriteBytes("something");
NUnit.Framework.Assert.IsTrue(HasLease(cluster, a));
NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b));
DataOutputStream b_out = fs.Create(b);
b_out.WriteBytes("something");
NUnit.Framework.Assert.IsTrue(HasLease(cluster, a));
NUnit.Framework.Assert.IsTrue(HasLease(cluster, b));
a_out.Close();
b_out.Close();
NUnit.Framework.Assert.IsTrue(!HasLease(cluster, a));
NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b));
fs.Delete(dir, true);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public override void StoreTokenMasterKey(DelegationKey masterKey)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing master key " + masterKey.GetKeyId());
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
masterKey.Write(dataStream);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
string dbKey = GetTokenMasterKeyDatabaseKey(masterKey);
try
{
db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
/// <exception cref="System.IO.IOException"/>
internal static void CreateFile(FileSystem fs, Path f)
{
DataOutputStream a_out = fs.Create(f);
a_out.WriteBytes("something");
a_out.Close();
}
public byte[] toByteArray() {
short datalen = 0;
byte[] data = null;
byte[] bytes = null;
byte[] byteNew = null;
try {
if (dos != null) {
dos.Flush();
data = ms.ToArray();
datalen = (short)data.Length;
dos.Close();
}
MemoryStream bos1 = new MemoryStream(datalen + 3);
DataOutputStream dos1 = new DataOutputStream(new BinaryWriterIns(bos1));
dos1.WriteByteNew(command);
dos1.WriteShort(datalen);
if (datalen > 0) {
dos1.Write(data);
}
bytes = bos1.ToArray();
byteNew = new byte[bytes.Length - 3];
int n = byteNew.Length;
Array.Copy(bytes, 3, byteNew, 0, n);
byteNew[0] = (byte)command;
dos1.Close();
}
catch (IOException e) {
Debug.Log(e.ToString());
}
return byteNew;
}
/// <summary>Assert MapWritable does not grow across calls to readFields.</summary>
/// <exception cref="System.Exception"/>
/// <seealso><a href="https://issues.apache.org/jira/browse/HADOOP-2244">HADOOP-2244</a>
/// </seealso>
public virtual void TestMultipleCallsToReadFieldsAreSafe()
{
// Create an instance and add a key/value.
MapWritable m = new MapWritable();
Text t = new Text(GetName());
m[t] = t;
// Get current size of map. Key values are 't'.
int count = m.Count;
// Now serialize... save off the bytes.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
m.Write(dos);
dos.Close();
// Now add new values to the MapWritable.
m[new Text("key1")] = new Text("value1");
m[new Text("key2")] = new Text("value2");
// Now deserialize the original MapWritable. Ensure count and key values
// match original state.
ByteArrayInputStream bais = new ByteArrayInputStream(baos.ToByteArray());
DataInputStream dis = new DataInputStream(bais);
m.ReadFields(dis);
Assert.Equal(count, m.Count);
Assert.True(m[t].Equals(t));
dis.Close();
}
/// <exception cref="System.IO.IOException"/>
public static void SaveFloatArr(DataOutputStream rf, float[] arr)
{
rf.WriteInt(arr.Length);
byte[] lArr = FloatArrToByteArr(arr);
rf.Write(lArr);
rf.Close();
}
/// <exception cref="System.IO.IOException"/>
public static void SaveDoubleArr(DataOutputStream rf, double[] arr)
{
rf.WriteInt(arr.Length);
byte[] lArr = DoubleArrToByteArr(arr);
rf.Write(lArr);
rf.Close();
}
private void WriteToDisk(System.IO.IsolatedStorage.IsolatedStorageFile iso, string storeFile)
{
DataOutputStream dos = FileUtils.WriteIsolatedStorageFileToDataInput(iso, storeFile);
try
{
dos.WriteUTF(HEADER);
dos.WriteInt(nextRecordId);
dos.WriteInt(records.Count);
for (int i = 0; i < records.Count; i++)
{
RecordItem ri = records[i];
long pSize = ri.data.Length;
int pId = ri.id;
dos.WriteLong(pSize);
dos.WriteInt(pId);
dos.Write(ri.data);
}
}
catch (Exception e)
{
throw new RecordStoreException("Error writing store to disk: " + e.StackTrace);
}
finally
{
if (dos != null)
{
dos.Close();
}
dos = null;
}
}
public virtual void TestFailureWriteRecordAfterMetaBlock()
{
if (skip)
{
return;
}
// write a key/value first
writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString
("valueX"));
// create a new metablock
DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm
.Gz.GetName());
outMeta.Write(123);
outMeta.Write(Runtime.GetBytesForString("dummy"));
outMeta.Close();
// add more key/value
try
{
writer.Append(Runtime.GetBytesForString("keyY"), Runtime.GetBytesForString
("valueY"));
NUnit.Framework.Assert.Fail("Cannot add key/value after start adding meta blocks."
);
}
catch (Exception)
{
}
// noop, expecting exceptions
CloseOutput();
}
/// <exception cref="System.IO.IOException"/>
public static Job CreateJob(Configuration conf, Path inDir, Path outDir, int numInputFiles
, int numReds, string input)
{
Job job = Job.GetInstance(conf);
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (fs.Exists(inDir))
{
fs.Delete(inDir, true);
}
fs.Mkdirs(inDir);
for (int i = 0; i < numInputFiles; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
FileInputFormat.SetInputPaths(job, inDir);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetNumReduceTasks(numReds);
return(job);
}
public static async Task WriteFile(Context context, Uri uri, byte[] data)
{
// Run backup on separate thread, file writing on the main thread fails when using Nextcloud
await Task.Run(async delegate
{
// This is the only way of reliably writing binary files using SAF on Xamarin.
// A file output stream will usually create 0 byte files on virtual storage such as Google Drive
Stream output = null;
DataOutputStream dataStream = null;
try
{
output = context.ContentResolver.OpenOutputStream(uri);
dataStream = new DataOutputStream(output);
await dataStream.WriteAsync(data);
await dataStream.FlushAsync();
}
finally
{
dataStream?.Close();
output?.Close();
}
});
}
// configure a job
/// <exception cref="System.IO.IOException"/>
private void Configure(JobConf conf, Path inDir, Path outDir, string input, Type
map, Type reduce)
{
// set up the input file system and write input text.
FileSystem inFs = inDir.GetFileSystem(conf);
FileSystem outFs = outDir.GetFileSystem(conf);
outFs.Delete(outDir, true);
if (!inFs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
{
// write input into input file
DataOutputStream file = inFs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
}
// configure the mapred Job which creates a tempfile in map.
conf.SetJobName("testmap");
conf.SetMapperClass(map);
conf.SetReducerClass(reduce);
conf.SetNumMapTasks(1);
conf.SetNumReduceTasks(0);
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
string TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).ToString
().Replace(' ', '+');
conf.Set("test.build.data", TestRootDir);
}
/// <exception cref="System.IO.IOException"/>
public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate
)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Storing token " + tokenId.GetSequenceNumber());
}
ByteArrayOutputStream memStream = new ByteArrayOutputStream();
DataOutputStream dataStream = new DataOutputStream(memStream);
try
{
tokenId.Write(dataStream);
dataStream.WriteLong(renewDate);
dataStream.Close();
dataStream = null;
}
finally
{
IOUtils.Cleanup(Log, dataStream);
}
string dbKey = GetTokenDatabaseKey(tokenId);
try
{
db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray());
}
catch (DBException e)
{
throw new IOException(e);
}
}
public byte[] toByteArray()
{
short datalen = 0;
byte[] data = null;
byte[] bytes = null;
byte[] byteNew = null;
try {
if (dos != null)
{
dos.Flush();
data = ms.ToArray();
datalen = (short)data.Length;
dos.Close();
}
MemoryStream bos1 = new MemoryStream(datalen + 3);
DataOutputStream dos1 = new DataOutputStream(new BinaryWriterIns(bos1));
dos1.WriteByteNew(command);
dos1.WriteShort(datalen);
if (datalen > 0)
{
dos1.Write(data);
}
bytes = bos1.ToArray();
byteNew = new byte[bytes.Length - 3];
int n = byteNew.Length;
Array.Copy(bytes, 3, byteNew, 0, n);
byteNew[0] = (byte)command;
dos1.Close();
}
catch (IOException e) {
Debug.Log(e.ToString());
}
return(byteNew);
}
/// <exception cref="System.IO.IOException"/>
public override void Close(TaskAttemptContext context)
{
lock (this)
{
@out.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public TestFileDescriptorPair()
{
fis = new FileInputStream[2];
for (int i = 0; i < 2; i++)
{
string name = dir.GetDir() + "/file" + i;
FileOutputStream fos = new FileOutputStream(name);
if (i == 0)
{
// write 'data' file
fos.Write(1);
}
else
{
// write 'metadata' file
BlockMetadataHeader header = new BlockMetadataHeader((short)1, DataChecksum.NewDataChecksum
(DataChecksum.Type.Null, 4));
DataOutputStream dos = new DataOutputStream(fos);
BlockMetadataHeader.WriteHeader(dos, header);
dos.Close();
}
fos.Close();
fis[i] = new FileInputStream(name);
}
}
private async Task DoBackup(Uri uri, string password)
{
var backup = new Backup(
_authSource.GetAll(),
_categorySource.GetAll(),
_authSource.CategoryBindings,
_customIconSource.GetAll()
);
var dataToWrite = backup.ToBytes(password);
// This is the only way of reliably writing files using SAF on Xamarin.
// A file output stream will usually create 0 byte files on virtual storage such as Google Drive
var output = ContentResolver.OpenOutputStream(uri, "rwt");
var dataStream = new DataOutputStream(output);
try
{
await dataStream.WriteAsync(dataToWrite);
await dataStream.FlushAsync();
}
finally
{
dataStream.Close();
output.Close();
}
}
/// <summary>
/// Closes, if necessary, the output stream.
/// </summary>
public void Close()
{
if (mDataOutputStream != null)
{
mDataOutputStream.Close();
}
}
public virtual void Write(float value) {
if (disposed)
throw new ObjectDisposedException ("BinaryWriter", "Cannot write to a closed BinaryWriter");
var byteArrayOutputStream = new ByteArrayOutputStream();
var dataOutputStream = new DataOutputStream(byteArrayOutputStream);
dataOutputStream.WriteFloat(value);
byteArrayOutputStream.Flush();
WriteSwapped(byteArrayOutputStream.ToByteArray(), 4);
byteArrayOutputStream.Close();
dataOutputStream.Close();
}
