private SlicedBlobReader(Stream outputStream, FPTag t, int threads)
{
int sliceSize = (int)(t.BlobSize / threads);
int lastSlice = (int)(t.BlobSize - (sliceSize * (threads - 1)));
int i, numBytes = sliceSize;
Thread[] readThread = new Thread[threads];
for (i = 0; i < threads; i++)
{
if (i == (threads - 1))
{
numBytes = lastSlice;
}
readThread[i] = new PartialBlobReadThread(t, outputStream, i * sliceSize, numBytes);
readThread[i].Start();
}
// Wait for all the threads to complete
for (i = 0; i < threads; i++)
{
readThread[i].Join();
}
FPLogger.ConsoleMessage("\n** finished");
}
private SlicedBlobWriter(Stream inputStream, FPTag tag, int threads)
{
int sliceSize = (int)(inputStream.Length / threads);
int lastSlice = (int)(inputStream.Length - (sliceSize * (threads - 1)));
int numBytes = sliceSize;
int i;
Thread[] writeThread = new Thread[threads];
//PartialBlobWriteStream[] slices = new PartialBlobWriteStream[threads];
for (i = 0; i < threads; i++)
{
if (i == (threads - 1))
{
numBytes = lastSlice;
}
writeThread[i] = new PartialBlobWriteThread(tag, i, inputStream, (i * sliceSize), numBytes);
writeThread[i].Start();
}
for (i = 0; i < threads; i++)
{
writeThread[i].Join();
}
FPLogger.ConsoleMessage("\nFinished");
}
// Standard ctor - Stream to be used to buffer, the Tag we read the data from
public BlobReadStream(Stream s, FPTag t)
: base(s, t)
{
myCallbacks = new Callbacks(holdingArea);
fpStream = new FPGenericStream(holdingArea, FPStream.StreamDirection.OutputFromCentera, myCallbacks, IntPtr.Zero);
fpStream.StreamLen = t.BlobSize;
FPLogger.ConsoleMessage("\nStarting blob read thread");
blobThread = new Thread(delegate() { t.BlobRead(fpStream); });
blobThread.Name = "BlobReadThread";
blobThread.Start();
}
static void Main(string[] args)
{
try
{
FPLogger.ConsoleMessage("\nCluster to connect to [" + defaultCluster + "] : ");
String clusterAddress = System.Console.ReadLine();
if ("" == clusterAddress)
{
clusterAddress = defaultCluster;
}
FPLogger.ConsoleMessage("\nEnter the name of the file containing the clip to be restored: ");
String fileName = Console.ReadLine();
if (!fileName.EndsWith(".xml"))
{
FPLogger.ConsoleMessage("\nInvalid file name format - should be <clipID>.xml");
return;
}
String clipID = fileName.Substring(0, fileName.Length - 4);
FPPool thePool = new FPPool(clusterAddress);
FPStream fpStream = new FPStream(fileName, 16 * 1024);
// Create the clip
FPClip clipRef = thePool.ClipRawOpen(clipID, fpStream, FPMisc.OPTION_DEFAULT_OPTIONS);
FPTag t = clipRef.NextTag;
fpStream = new FPStream(clipID + ".Tag.0", 16 * 1024);
t.BlobWrite(fpStream, FPMisc.OPTION_DEFAULT_OPTIONS);
fpStream.Close();
t.Close();
t = clipRef.NextTag;
fpStream = new FPStream(clipID + ".Tag.1", 16 * 1024);
t.BlobWrite(fpStream, FPMisc.OPTION_DEFAULT_OPTIONS);
fpStream.Close();
t.Close();
clipRef.Write();
clipRef.Close();
thePool.Close();
FPLogger.ConsoleMessage("\nClip and Blobs successfully restored to Cluster");
}
catch (FPLibraryException e)
{
ErrorInfo err = e.errorInfo;
FPLogger.ConsoleMessage("\nException thrown in FP Library: Error " + err.error + " " + err.message);
}
}
internal PartialBlobWriteThread(FPTag t, int sequenceNum, Stream s, int offset, int numBytes)
: base(t, sequenceNum)
{
myWriteThread = new Thread(delegate()
{
BinaryReader myReader = new BinaryReader(new FPPartialInputStream(s, offset, numBytes));
BinaryWriter myWriter = new BinaryWriter(this);
FPLogger.ConsoleMessage("\nStarted sliced write thread " + GetHashCode() + " at " + offset + " size " + numBytes);
myWriter.Write(myReader.ReadBytes(numBytes));
myWriter.Flush();
Close();
});
}
// Standard ctor - Stream to be used to buffer, the Tag we write the data to
public BlobWriteStream(Stream s, FPTag t)
: base(s, t)
{
myCallbacks = new Callbacks(holdingArea);
fpStream = new FPGenericStream(holdingArea, FPStream.StreamDirection.InputToCentera, myCallbacks, (IntPtr)this.GetHashCode());
blobThread = new Thread(delegate() { theTag.BlobWrite(fpStream); });
blobThread.Name = "BlobWriteThread";
FPLogger.ConsoleMessage("\nStarted stream " + this.GetHashCode() + " blob write thread " + blobThread.GetHashCode());
// We keep track of the current "open" streams being used in a hashtable
// When user closes the stream we remove this to signal the callback that we are done writing
openStreams.Add(this.GetHashCode(), blobThread.GetHashCode());
}
internal PartialBlobReadThread(FPTag t, Stream s, int offset, int numBytes)
: base(t, offset, numBytes)
{
myReadThread = new Thread(delegate()
{
BinaryReader myReader = new BinaryReader(this);
BinaryWriter myWriter = new BinaryWriter(new FPPartialOutputStream(s, offset, numBytes));
FPLogger.ConsoleMessage("\nStarted sliced read thread " + GetHashCode() + " at " + offset + " size " + numBytes);
// Wait until we have all the read content back in the buffer
Join();
myWriter.Write(myReader.ReadBytes(numBytes));
Close();
});
}
// Standard ctor - as for BlobReadStream but also specify the offset and number of bytes for the partial read.
public PartialBlobReadStream(Stream s, FPTag t, long offset, long numBytes)
{
// We cannot call the base constructor because we need use the additional offset and numBytes parameters
// Even if we split out some of the functionality using a virtual init method with no
// parameters, these paraameters will not be initialized when the virtual function was called
// by the base constructor. So we just have to duplicate the (small)initialization code.
holdingArea = s;
theTag = t;
myCallbacks = new Callbacks(holdingArea, 16 * 1024);
fpStream = new FPGenericStream(holdingArea, FPStream.StreamDirection.OutputFromCentera, myCallbacks, IntPtr.Zero);
fpStream.StreamLen = numBytes;
blobThread = new Thread(delegate() { theTag.BlobReadPartial(fpStream, offset, numBytes); });
blobThread.Name = "PartialBlobReadThread";
blobThread.Start();
}
// Standard ctor - as for BlobWriteStream but also specify the sequence number of the segment.
public PartialBlobWriteStream(Stream s, FPTag t, int id)
{
// We cannot call the base constructor because we need use the additional id parameter
// Even if we split out some of the functionality using a virtual init method with no
// parameters, the sequenceId will not be initialized when the virtual function was called
// by the base constructor. So we just have to duplicate the (small)initialization code.
holdingArea = s;
theTag = t;
sequenceId = id;
myCallbacks = new Callbacks(holdingArea);
fpStream = new FPGenericStream(holdingArea, FPStream.StreamDirection.InputToCentera, myCallbacks, (IntPtr)this.GetHashCode());
FPLogger.ConsoleMessage("\n ** Partial blob write stream with sequence id = " + sequenceId);
blobThread = new Thread(delegate() { theTag.BlobWritePartial(fpStream, sequenceId); });
blobThread.Name = "PartialBlobWriteThread";
}
public static void Instance(Stream inputStream, FPTag tag, int threads)
{
SlicedBlobWriter writer = new SlicedBlobWriter(inputStream, tag, threads);
}
// Default ctor using a MemoryStream for the buffer
public PartialBlobWriteStream(FPTag t, int id) : this(new MemoryStream(), t, id)
{
}
static void Main(string[] args)
{
IntPtr userData = new IntPtr(0);
string clipID;
BinaryReader myReader;
BinaryWriter myWriter;
BlobStream myStream;
// For thsi test we are using a file as the input. *Any* stream could be used.
// This would be your test PDF - in a file or memory stream.
FileStream inputStream = new FileStream("e:\\testfile", FileMode.Open);
FPLogger.ConsoleMessage("\nInput file length " + inputStream.Length);
try
{
FPLogger log = new FPLogger();
log.LogPath = "C:\\GenStreamsLog.txt";
log.Start();
// Create a test clip demonstrating how to write a blob using a stream
FPPool myPool = new FPPool("cse1.centera.lab.emc.com");
FPClip myClip = myPool.ClipCreate("GenericStreamWrite_testClip");
FPTag myTag = myClip.AddTag("testTag");
// Simple test of the BlobWriteStream - write contents of a local file
// to Centera using a StreamReader and StreamWriter
// We are using a temporary MemoryStream for our staging area
myStream = new BlobWriteStream(myTag);
myReader = new BinaryReader(inputStream);
myWriter = new BinaryWriter(myStream);
long writeTime1 = DateTime.Now.Ticks;
myWriter.Write(myReader.ReadBytes((int)inputStream.Length));
myWriter.Flush();
((BlobWriteStream)myStream).Close(); // If we don't do this it will stay open forever with keep alives!
writeTime1 = DateTime.Now.Ticks - writeTime1;
// If we assign values to existing FP objects we must explicitly close
// them first to avoid potential "Object In Use" errors on future closure
myTag.Close();
myTag = myClip.AddTag("SlicedBlob");
inputStream.Seek(0, SeekOrigin.Begin);
long writeTime = DateTime.Now.Ticks;
SlicedBlobWriter.Instance(inputStream, myTag, 8);
writeTime = DateTime.Now.Ticks - writeTime;
myTag.Close();
clipID = myClip.Write();
myClip.Close();
FPLogger.ConsoleMessage("\nGenericStreamWrite test succeeded " + clipID);
// Now we will test reading it back from the Centera {
myClip = myPool.ClipOpen(clipID, FPMisc.OPEN_ASTREE);
myTag = myClip.NextTag;
// Here we create a stream to read back the blob
long readTime1 = DateTime.Now.Ticks;
myStream = new BlobReadStream(myTag);
// This is only to test to verify when we read the content back it is the same as the
// input file i.e. we are going to store it on the file system and then use file system tools
// to compare the files
myReader = new BinaryReader(myStream);
myWriter = new BinaryWriter(new FileStream("c:\\testfile2.out", FileMode.Create));
((BlobReadStream)myStream).Join();
myWriter.Write(myReader.ReadBytes((int)myTag.BlobSize));
myWriter.Flush();
myStream.Close();
readTime1 = DateTime.Now.Ticks - readTime1;
myTag.Close();
// This shows how to read content back using multiple threads
myTag = myClip.NextTag;
long readTime = DateTime.Now.Ticks;
SlicedBlobReader.Instance(new FileStream("c:\\testfile2.out.sliced", FileMode.Create), myTag, 8);
readTime = DateTime.Now.Ticks - readTime;
myTag.Close();
myClip.Close();
myPool.Close();
FPLogger.ConsoleMessage("\nSerial tests - Write time: " + TimeSpan.FromTicks(writeTime1) + " Read time " + TimeSpan.FromTicks(readTime1));
FPLogger.ConsoleMessage("\nMulti threaded tests - Write time: " + TimeSpan.FromTicks(writeTime) + " Read time " + TimeSpan.FromTicks(readTime));
}
catch (FPLibraryException e)
{
ErrorInfo err = e.errorInfo;
FPLogger.ConsoleMessage("\nException thrown in FP Library: Error " + err.error + " " + err.message);
}
}
// Default ctor that uses a MemoryStream for the temporary buffer
public BlobReadStream(FPTag t) : this(new MemoryStream(), t)
{
}
// Default ctor that uses a MemoryStream for the temporary buffer
public BlobWriteStream(FPTag t) : this(new MemoryStream(), t)
{
}
static void Main(string[] args)
{
try
{
FPLogger.ConsoleMessage("\nCluster to connect to [" + defaultCluster + "] :");
String clusterAddress = System.Console.ReadLine();
if ("" == clusterAddress)
{
clusterAddress = defaultCluster;
}
FPLogger.ConsoleMessage("\nEnter the CA of the content to be restored: ");
String clipID = System.Console.ReadLine();
FPPool thePool = new FPPool(clusterAddress);
FPClip clipRef = new FPClip(thePool, clipID, FPMisc.OPEN_ASTREE);
FPLogger.ConsoleMessage("\nThe clip retention expires on " + clipRef.RetentionExpiry);
// Iterate across the clip metadata
foreach (FPAttribute attr in clipRef.Attributes)
{
FPLogger.ConsoleMessage(attr.ToString() + "\n");
}
// Iterate across the Tag (and their Attr) collections
foreach (FPTag tg in clipRef.Tags)
{
FPLogger.ConsoleMessage(tg.ToString() + "\n");
foreach (FPAttribute a in tg.Attributes)
{
FPLogger.ConsoleMessage(a.ToString() + "\n");
}
}
FPTag t = (FPTag)clipRef.Tags[0];
if (t.ToString() == "StoreContentObject")
{
String outfile = clipRef.ClipID + ".Tag." + 0;
FPStream streamRef = new FPStream(outfile, "wb");
t.BlobRead(streamRef, FPMisc.OPTION_DEFAULT_OPTIONS);
streamRef.Close();
FPLogger.ConsoleMessage("\nThe Blob has been stored into " + outfile);
t.Close();
t = (FPTag)clipRef.Tags[1];
//Do the same for the second tag
outfile = clipRef.ClipID + ".Tag." + 1;
streamRef = new FPStream(outfile, "wb");
t.BlobRead(streamRef, FPMisc.OPTION_DEFAULT_OPTIONS);
streamRef.Close();
FPLogger.ConsoleMessage("\nThe Blob has been stored into " + outfile);
FPLogger.ConsoleMessage("\nThe CDF has been saved to " + clipRef.ClipID + ".xml");
streamRef = new FPStream(clipRef.ClipID + ".xml", "wb");
clipRef.RawRead(streamRef);
// We could retrieve the blob into a buffer - we know we stored
// a string in the StoreContent sample so let's get it back..
int blobSize = (int)t.BlobSize;
UTF8Encoding converter = new UTF8Encoding();
byte[] buffer = new byte[blobSize];
IntPtr source = Marshal.AllocHGlobal(blobSize);
streamRef = new FPStream(source, blobSize, FPStream.StreamDirection.OutputFromCentera);
t.BlobRead(streamRef);
Marshal.Copy(source, buffer, 0, blobSize);
Marshal.FreeHGlobal(source);
FPLogger.ConsoleMessage("\n" + converter.GetString(buffer));
streamRef.Close();
}
else
{
FPLogger.ConsoleMessage("\nApplication Error: Not A C-Clip Created By StoreContent Example");
}
}
catch (FPLibraryException e)
{
ErrorInfo err = e.errorInfo;
FPLogger.ConsoleMessage("\nException thrown in FP Library: Error " + err.error + " " + err.message);
}
}
public static void Instance(Stream outputStream, FPTag tag, int threads)
{
SlicedBlobReader writer = new SlicedBlobReader(outputStream, tag, threads);
}
// Default ctor using a MemoryStream for the buffer
public PartialBlobReadStream(FPTag t, long offset, long numBytes) : this(new MemoryStream(), t, offset, numBytes)
{
}
protected FPTag theTag; // The Tag we are writing to or reading from
protected BlobStream(Stream s, FPTag t)
{
holdingArea = s;
theTag = t;
}
static void Main(string[] args)
{
try
{
FPLogger.ConsoleMessage("\nCluster to connect to [" + defaultCluster + "] : ");
String clusterAddress = System.Console.ReadLine();
if ("" == clusterAddress)
{
clusterAddress = defaultCluster;
}
UTF8Encoding converter2 = new UTF8Encoding();
// We are going to create a filename containing Chinese characters
byte[] inputFileName = { 0xE6, 0x98, 0x8E, 0xE5, 0xA4, 0xA9, 0x2E, 0x74, 0x78, 0x74 };
string fileName = System.Text.Encoding.UTF8.GetString(inputFileName);
File.WriteAllText(fileName, "Test file with wide characters in the file name\n");
FPLogger.ConsoleMessage("\nEnter the threshold to use for Embedded Blobs: ");
String blobString = System.Console.ReadLine();
if (blobString == "")
{
blobString = "0";
}
int blobThreshold = Int32.Parse(blobString);
if (blobThreshold > 0)
{
FPLogger.ConsoleMessage("\nContent less than " + blobString + " bytes will be embedded in the CDF.");
}
else
{
FPLogger.ConsoleMessage("\nContent will never be embedded in the CDF.");
}
FPPool.EmbeddedBlobThreshold = blobThreshold;
FPPool thePool = new FPPool(clusterAddress);
// Create the clip
FPClip clipRef = new FPClip(thePool, "UTF8_Example");
clipRef.SetAttribute("OriginalFilename", fileName);
FPTag fileTag = clipRef.AddTag("WideFilenameTag");
fileTag.SetAttribute("filename", fileName);
// Write the content of the file to the Centera
FPStream streamRef = new FPWideFilenameStream(fileName);
fileTag.BlobWrite(streamRef, FPMisc.OPTION_CLIENT_CALCID);
streamRef.Close();
long blobSize = fileTag.BlobSize;
if (blobThreshold > 0 && blobSize < blobThreshold)
{
FPLogger.ConsoleMessage("\nContent was embedded in the CDF as size "
+ blobSize
+ " is less than the threshold.");
}
fileTag.Close();
fileTag = clipRef.AddTag("TestTag");
fileTag.SetAttribute("fileCabinetGuid", "52d9cf57-7261-472a-b6d9-a8cdd30d1d27");
fileTag.SetAttribute("attr2", "a second attribute");
fileTag.SetAttribute("anotherattr", "the third attribute");
/* Or you can write from a memory buffer holding the data
* The exampe uses a byte array containing a string
*/
UTF8Encoding converter = new UTF8Encoding();
byte[] rawBuffer = converter.GetBytes("This is a test string to illustrate how to interface to Centera using a buffer stream");
IntPtr source = Marshal.AllocHGlobal(rawBuffer.Length);
Marshal.Copy(rawBuffer, 0, source, rawBuffer.Length);
streamRef = new FPStream(source, rawBuffer.Length, FPStream.StreamDirection.InputToCentera);
fileTag.BlobWrite(streamRef, FPMisc.OPTION_CLIENT_CALCID);
streamRef.Close();
Marshal.FreeHGlobal(source);
fileTag.Close();
fileTag = clipRef.AddTag("TestTag2");
fileTag.SetAttribute("filename", fileName);
// We will only store a fragment of the file - "length" characters from position "offset"
long offset = 5;
long length = 30;
streamRef = new FPWideFilenameStream(fileName, offset, length);
fileTag.BlobWrite(streamRef, FPMisc.OPTION_CLIENT_CALCID);
streamRef.Close();
// And this is how we read back at most "length" characters into a file at position "offset"
// We can also specify that we don't want the created file to be larger than "maxFileSize"
long maxFileSize = 20;
streamRef = new FPWideFilenameStream(fileName + ".partial", FileMode.OpenOrCreate, offset, length, maxFileSize);
fileTag.BlobRead(streamRef);
streamRef.Close();
String clipID = clipRef.Write();
/* Write the Clip ID to the output file, "inputFileName.clipID" */
FPLogger.ConsoleMessage("\nThe C-Clip ID of the content is " + clipID);
FileStream outFile = new FileStream(fileName + ".clipID", FileMode.Create);
BinaryWriter outWriter = new BinaryWriter(outFile);
outWriter.Write(clipID.ToString() + "\n");
outWriter.Close();
outFile.Close();
fileTag.Close();
clipRef.Close();
thePool.Close();
}
catch (FPLibraryException e)
{
ErrorInfo err = e.errorInfo;
FPLogger.ConsoleMessage("\nException thrown in FP Library: Error " + err.error + " " + err.message);
}
}
