// noop, expecting an exception /// <exception cref="System.IO.IOException"/> public virtual void TestFailureCloseKeyStreamManyTimesInWriter() { if (skip) { return; } DataOutputStream outKey = writer.PrepareAppendKey(4); try { outKey.Write(Runtime.GetBytesForString("key0")); outKey.Close(); } catch (Exception) { } finally { // noop, expecting an exception try { outKey.Close(); } catch (Exception) { } } // no-op outKey.Close(); outKey.Close(); Assert.True("Multiple close should have no effect.", true); }
/// <exception cref="System.Exception"/> private static void UploadEmptyContainerLogIntoRemoteDir(UserGroupInformation ugi , Configuration configuration, IList <string> rootLogDirs, NodeId nodeId, ContainerId containerId, Path appDir, FileSystem fs) { Path path = new Path(appDir, LogAggregationUtils.GetNodeString(nodeId) + Runtime. CurrentTimeMillis()); AggregatedLogFormat.LogWriter writer = new AggregatedLogFormat.LogWriter(configuration , path, ugi); writer.WriteApplicationOwner(ugi.GetUserName()); IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType , string>(); appAcls[ApplicationAccessType.ViewApp] = ugi.GetUserName(); writer.WriteApplicationACLs(appAcls); DataOutputStream @out = writer.GetWriter().PrepareAppendKey(-1); new AggregatedLogFormat.LogKey(containerId).Write(@out); @out.Close(); @out = writer.GetWriter().PrepareAppendValue(-1); new AggregatedLogFormat.LogValue(rootLogDirs, containerId, UserGroupInformation.GetCurrentUser ().GetShortUserName()).Write(@out, new HashSet <FilePath>()); @out.Close(); writer.Close(); }
// noop, expecting an exception /// <exception cref="System.IO.IOException"/> public virtual void TestFailureValueTooLong() { if (skip) { return; } DataOutputStream outKey = writer.PrepareAppendKey(4); outKey.Write(Runtime.GetBytesForString("key0")); outKey.Close(); DataOutputStream outValue = writer.PrepareAppendValue(3); try { outValue.Write(Runtime.GetBytesForString("value0")); outValue.Close(); NUnit.Framework.Assert.Fail("Value is longer than expected."); } catch (Exception) { } // noop, expecting an exception try { outKey.Close(); outKey.Close(); } catch (Exception) { NUnit.Framework.Assert.Fail("Second or more close() should have no effect."); } }
/// <exception cref="System.IO.IOException"/> public virtual void WriteApplicationOwner(string user) { DataOutputStream @out = this.writer.PrepareAppendKey(-1); ApplicationOwnerKey.Write(@out); @out.Close(); @out = this.writer.PrepareAppendValue(-1); @out.WriteUTF(user); @out.Close(); }
/// <exception cref="System.IO.IOException"/> private void WriteVersion() { DataOutputStream @out = this.writer.PrepareAppendKey(-1); VersionKey.Write(@out); @out.Close(); @out = this.writer.PrepareAppendValue(-1); @out.WriteInt(Version); @out.Close(); }
/// <exception cref="System.IO.IOException"/> public virtual void WriteApplicationACLs(IDictionary <ApplicationAccessType, string > appAcls) { DataOutputStream @out = this.writer.PrepareAppendKey(-1); ApplicationAclKey.Write(@out); @out.Close(); @out = this.writer.PrepareAppendValue(-1); foreach (KeyValuePair <ApplicationAccessType, string> entry in appAcls) { @out.WriteUTF(entry.Key.ToString()); @out.WriteUTF(entry.Value); } @out.Close(); }
/// <exception cref="System.IO.IOException"/> private int WritePrepWithUnkownLength(TFile.Writer writer, int start, int n) { for (int i = start; i < (start + n); i++) { DataOutputStream @out = writer.PrepareAppendKey(-1); string localKey = string.Format(localFormatter, i); @out.Write(Runtime.GetBytesForString(localKey)); @out.Close(); string value = "value" + localKey; @out = writer.PrepareAppendValue(-1); @out.Write(Runtime.GetBytesForString(value)); @out.Close(); } return(start + n); }
public virtual void TestFailureGetNonExistentMetaBlock() { if (skip) { return; } writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString ("valueX")); // create a new metablock DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm .Gz.GetName()); outMeta.Write(123); outMeta.Write(Runtime.GetBytesForString("foo")); outMeta.Close(); CloseOutput(); TFile.Reader reader = new TFile.Reader(fs.Open(path), fs.GetFileStatus(path).GetLen (), conf); DataInputStream mb = reader.GetMetaBlock("testX"); NUnit.Framework.Assert.IsNotNull(mb); mb.Close(); try { DataInputStream mbBad = reader.GetMetaBlock("testY"); NUnit.Framework.Assert.Fail("Error on handling non-existent metablocks."); } catch (Exception) { } // noop, expecting exceptions reader.Close(); }
private static void CreateBinaryTokenFile(Configuration conf) { // Fetch delegation tokens and store in binary token file. try { Credentials cred1 = new Credentials(); Credentials cred2 = new Credentials(); TokenCache.ObtainTokensForNamenodesInternal(cred1, new Path[] { p1 }, conf); foreach (Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> t in cred1.GetAllTokens ()) { cred2.AddToken(new Text(DelegationTokenKey), t); } DataOutputStream os = new DataOutputStream(new FileOutputStream(binaryTokenFileName .ToString())); try { cred2.WriteTokenStorageToStream(os); } finally { os.Close(); } } catch (IOException e) { NUnit.Framework.Assert.Fail("Exception " + e); } }
public virtual void TestFailureWriteMetaBlocksWithSameName() { if (skip) { return; } writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString ("valueX")); // create a new metablock DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm .Gz.GetName()); outMeta.Write(123); outMeta.Write(Runtime.GetBytesForString("foo")); outMeta.Close(); // add the same metablock try { writer.PrepareMetaBlock("testX", Compression.Algorithm.Gz.GetName()); NUnit.Framework.Assert.Fail("Cannot create metablocks with the same name."); } catch (Exception) { } // noop, expecting exceptions CloseOutput(); }
public virtual void TestLocalFileCorruption() { Configuration conf = new HdfsConfiguration(); Path file = new Path(PathUtils.GetTestDirName(GetType()), "corruptFile"); FileSystem fs = FileSystem.GetLocal(conf); DataOutputStream dos = fs.Create(file); dos.WriteBytes("original bytes"); dos.Close(); // Now deliberately corrupt the file dos = new DataOutputStream(new FileOutputStream(file.ToString())); dos.WriteBytes("corruption"); dos.Close(); // Now attempt to read the file DataInputStream dis = fs.Open(file, 512); try { System.Console.Out.WriteLine("A ChecksumException is expected to be logged."); dis.ReadByte(); } catch (ChecksumException) { } //expect this exception but let any NPE get thrown fs.Delete(file, true); }
/// <exception cref="System.IO.IOException"/> public override void StoreTokenMasterKey(DelegationKey key) { if (Log.IsDebugEnabled()) { Log.Debug("Storing master key " + key.GetKeyId()); } Path keyPath = new Path(tokenKeysStatePath, TokenMasterKeyFilePrefix + key.GetKeyId ()); if (fs.Exists(keyPath)) { throw new IOException(keyPath + " already exists"); } ByteArrayOutputStream memStream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(memStream); try { key.Write(dataStream); dataStream.Close(); dataStream = null; } finally { IOUtils.Cleanup(Log, dataStream); } CreateNewFile(keyPath, memStream.ToByteArray()); }
public virtual void TestStatusLimit() { Path test = new Path(testRootTempDir, "testStatusLimit"); Configuration conf = new Configuration(); Path inDir = new Path(test, "in"); Path outDir = new Path(test, "out"); FileSystem fs = FileSystem.Get(conf); if (fs.Exists(inDir)) { fs.Delete(inDir, true); } fs.Mkdirs(inDir); DataOutputStream file = fs.Create(new Path(inDir, "part-" + 0)); file.WriteBytes("testStatusLimit"); file.Close(); if (fs.Exists(outDir)) { fs.Delete(outDir, true); } Job job = Job.GetInstance(conf, "testStatusLimit"); job.SetMapperClass(typeof(TestReporter.StatusLimitMapper)); job.SetNumReduceTasks(0); FileInputFormat.AddInputPath(job, inDir); FileOutputFormat.SetOutputPath(job, outDir); job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful()); }
public virtual void TestLease() { MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); try { FileSystem fs = cluster.GetFileSystem(); NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir)); Path a = new Path(dir, "a"); Path b = new Path(dir, "b"); DataOutputStream a_out = fs.Create(a); a_out.WriteBytes("something"); NUnit.Framework.Assert.IsTrue(HasLease(cluster, a)); NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b)); DataOutputStream b_out = fs.Create(b); b_out.WriteBytes("something"); NUnit.Framework.Assert.IsTrue(HasLease(cluster, a)); NUnit.Framework.Assert.IsTrue(HasLease(cluster, b)); a_out.Close(); b_out.Close(); NUnit.Framework.Assert.IsTrue(!HasLease(cluster, a)); NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b)); fs.Delete(dir, true); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public override void StoreTokenMasterKey(DelegationKey masterKey) { if (Log.IsDebugEnabled()) { Log.Debug("Storing master key " + masterKey.GetKeyId()); } ByteArrayOutputStream memStream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(memStream); try { masterKey.Write(dataStream); dataStream.Close(); dataStream = null; } finally { IOUtils.Cleanup(Log, dataStream); } string dbKey = GetTokenMasterKeyDatabaseKey(masterKey); try { db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
/// <exception cref="System.IO.IOException"/> internal static void CreateFile(FileSystem fs, Path f) { DataOutputStream a_out = fs.Create(f); a_out.WriteBytes("something"); a_out.Close(); }
public byte[] toByteArray() { short datalen = 0; byte[] data = null; byte[] bytes = null; byte[] byteNew = null; try { if (dos != null) { dos.Flush(); data = ms.ToArray(); datalen = (short)data.Length; dos.Close(); } MemoryStream bos1 = new MemoryStream(datalen + 3); DataOutputStream dos1 = new DataOutputStream(new BinaryWriterIns(bos1)); dos1.WriteByteNew(command); dos1.WriteShort(datalen); if (datalen > 0) { dos1.Write(data); } bytes = bos1.ToArray(); byteNew = new byte[bytes.Length - 3]; int n = byteNew.Length; Array.Copy(bytes, 3, byteNew, 0, n); byteNew[0] = (byte)command; dos1.Close(); } catch (IOException e) { Debug.Log(e.ToString()); } return byteNew; }
/// <summary>Assert MapWritable does not grow across calls to readFields.</summary> /// <exception cref="System.Exception"/> /// <seealso><a href="https://issues.apache.org/jira/browse/HADOOP-2244">HADOOP-2244</a> /// </seealso> public virtual void TestMultipleCallsToReadFieldsAreSafe() { // Create an instance and add a key/value. MapWritable m = new MapWritable(); Text t = new Text(GetName()); m[t] = t; // Get current size of map. Key values are 't'. int count = m.Count; // Now serialize... save off the bytes. ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); m.Write(dos); dos.Close(); // Now add new values to the MapWritable. m[new Text("key1")] = new Text("value1"); m[new Text("key2")] = new Text("value2"); // Now deserialize the original MapWritable. Ensure count and key values // match original state. ByteArrayInputStream bais = new ByteArrayInputStream(baos.ToByteArray()); DataInputStream dis = new DataInputStream(bais); m.ReadFields(dis); Assert.Equal(count, m.Count); Assert.True(m[t].Equals(t)); dis.Close(); }
/// <exception cref="System.IO.IOException"/> public static void SaveFloatArr(DataOutputStream rf, float[] arr) { rf.WriteInt(arr.Length); byte[] lArr = FloatArrToByteArr(arr); rf.Write(lArr); rf.Close(); }
/// <exception cref="System.IO.IOException"/> public static void SaveDoubleArr(DataOutputStream rf, double[] arr) { rf.WriteInt(arr.Length); byte[] lArr = DoubleArrToByteArr(arr); rf.Write(lArr); rf.Close(); }
private void WriteToDisk(System.IO.IsolatedStorage.IsolatedStorageFile iso, string storeFile) { DataOutputStream dos = FileUtils.WriteIsolatedStorageFileToDataInput(iso, storeFile); try { dos.WriteUTF(HEADER); dos.WriteInt(nextRecordId); dos.WriteInt(records.Count); for (int i = 0; i < records.Count; i++) { RecordItem ri = records[i]; long pSize = ri.data.Length; int pId = ri.id; dos.WriteLong(pSize); dos.WriteInt(pId); dos.Write(ri.data); } } catch (Exception e) { throw new RecordStoreException("Error writing store to disk: " + e.StackTrace); } finally { if (dos != null) { dos.Close(); } dos = null; } }
public virtual void TestFailureWriteRecordAfterMetaBlock() { if (skip) { return; } // write a key/value first writer.Append(Runtime.GetBytesForString("keyX"), Runtime.GetBytesForString ("valueX")); // create a new metablock DataOutputStream outMeta = writer.PrepareMetaBlock("testX", Compression.Algorithm .Gz.GetName()); outMeta.Write(123); outMeta.Write(Runtime.GetBytesForString("dummy")); outMeta.Close(); // add more key/value try { writer.Append(Runtime.GetBytesForString("keyY"), Runtime.GetBytesForString ("valueY")); NUnit.Framework.Assert.Fail("Cannot add key/value after start adding meta blocks." ); } catch (Exception) { } // noop, expecting exceptions CloseOutput(); }
/// <exception cref="System.IO.IOException"/> public static Job CreateJob(Configuration conf, Path inDir, Path outDir, int numInputFiles , int numReds, string input) { Job job = Job.GetInstance(conf); FileSystem fs = FileSystem.Get(conf); if (fs.Exists(outDir)) { fs.Delete(outDir, true); } if (fs.Exists(inDir)) { fs.Delete(inDir, true); } fs.Mkdirs(inDir); for (int i = 0; i < numInputFiles; ++i) { DataOutputStream file = fs.Create(new Path(inDir, "part-" + i)); file.WriteBytes(input); file.Close(); } FileInputFormat.SetInputPaths(job, inDir); FileOutputFormat.SetOutputPath(job, outDir); job.SetNumReduceTasks(numReds); return(job); }
public static async Task WriteFile(Context context, Uri uri, byte[] data) { // Run backup on separate thread, file writing on the main thread fails when using Nextcloud await Task.Run(async delegate { // This is the only way of reliably writing binary files using SAF on Xamarin. // A file output stream will usually create 0 byte files on virtual storage such as Google Drive Stream output = null; DataOutputStream dataStream = null; try { output = context.ContentResolver.OpenOutputStream(uri); dataStream = new DataOutputStream(output); await dataStream.WriteAsync(data); await dataStream.FlushAsync(); } finally { dataStream?.Close(); output?.Close(); } }); }
// configure a job /// <exception cref="System.IO.IOException"/> private void Configure(JobConf conf, Path inDir, Path outDir, string input, Type map, Type reduce) { // set up the input file system and write input text. FileSystem inFs = inDir.GetFileSystem(conf); FileSystem outFs = outDir.GetFileSystem(conf); outFs.Delete(outDir, true); if (!inFs.Mkdirs(inDir)) { throw new IOException("Mkdirs failed to create " + inDir.ToString()); } { // write input into input file DataOutputStream file = inFs.Create(new Path(inDir, "part-0")); file.WriteBytes(input); file.Close(); } // configure the mapred Job which creates a tempfile in map. conf.SetJobName("testmap"); conf.SetMapperClass(map); conf.SetReducerClass(reduce); conf.SetNumMapTasks(1); conf.SetNumReduceTasks(0); FileInputFormat.SetInputPaths(conf, inDir); FileOutputFormat.SetOutputPath(conf, outDir); string TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).ToString ().Replace(' ', '+'); conf.Set("test.build.data", TestRootDir); }
/// <exception cref="System.IO.IOException"/> public override void StoreToken(MRDelegationTokenIdentifier tokenId, long renewDate ) { if (Log.IsDebugEnabled()) { Log.Debug("Storing token " + tokenId.GetSequenceNumber()); } ByteArrayOutputStream memStream = new ByteArrayOutputStream(); DataOutputStream dataStream = new DataOutputStream(memStream); try { tokenId.Write(dataStream); dataStream.WriteLong(renewDate); dataStream.Close(); dataStream = null; } finally { IOUtils.Cleanup(Log, dataStream); } string dbKey = GetTokenDatabaseKey(tokenId); try { db.Put(JniDBFactory.Bytes(dbKey), memStream.ToByteArray()); } catch (DBException e) { throw new IOException(e); } }
public byte[] toByteArray() { short datalen = 0; byte[] data = null; byte[] bytes = null; byte[] byteNew = null; try { if (dos != null) { dos.Flush(); data = ms.ToArray(); datalen = (short)data.Length; dos.Close(); } MemoryStream bos1 = new MemoryStream(datalen + 3); DataOutputStream dos1 = new DataOutputStream(new BinaryWriterIns(bos1)); dos1.WriteByteNew(command); dos1.WriteShort(datalen); if (datalen > 0) { dos1.Write(data); } bytes = bos1.ToArray(); byteNew = new byte[bytes.Length - 3]; int n = byteNew.Length; Array.Copy(bytes, 3, byteNew, 0, n); byteNew[0] = (byte)command; dos1.Close(); } catch (IOException e) { Debug.Log(e.ToString()); } return(byteNew); }
/// <exception cref="System.IO.IOException"/> public override void Close(TaskAttemptContext context) { lock (this) { @out.Close(); } }
/// <exception cref="System.IO.IOException"/> public TestFileDescriptorPair() { fis = new FileInputStream[2]; for (int i = 0; i < 2; i++) { string name = dir.GetDir() + "/file" + i; FileOutputStream fos = new FileOutputStream(name); if (i == 0) { // write 'data' file fos.Write(1); } else { // write 'metadata' file BlockMetadataHeader header = new BlockMetadataHeader((short)1, DataChecksum.NewDataChecksum (DataChecksum.Type.Null, 4)); DataOutputStream dos = new DataOutputStream(fos); BlockMetadataHeader.WriteHeader(dos, header); dos.Close(); } fos.Close(); fis[i] = new FileInputStream(name); } }
private async Task DoBackup(Uri uri, string password) { var backup = new Backup( _authSource.GetAll(), _categorySource.GetAll(), _authSource.CategoryBindings, _customIconSource.GetAll() ); var dataToWrite = backup.ToBytes(password); // This is the only way of reliably writing files using SAF on Xamarin. // A file output stream will usually create 0 byte files on virtual storage such as Google Drive var output = ContentResolver.OpenOutputStream(uri, "rwt"); var dataStream = new DataOutputStream(output); try { await dataStream.WriteAsync(dataToWrite); await dataStream.FlushAsync(); } finally { dataStream.Close(); output.Close(); } }
/// <summary> /// Closes, if necessary, the output stream. /// </summary> public void Close() { if (mDataOutputStream != null) { mDataOutputStream.Close(); } }
public virtual void Write(float value) { if (disposed) throw new ObjectDisposedException ("BinaryWriter", "Cannot write to a closed BinaryWriter"); var byteArrayOutputStream = new ByteArrayOutputStream(); var dataOutputStream = new DataOutputStream(byteArrayOutputStream); dataOutputStream.WriteFloat(value); byteArrayOutputStream.Flush(); WriteSwapped(byteArrayOutputStream.ToByteArray(), 4); byteArrayOutputStream.Close(); dataOutputStream.Close(); }