public TrinketProxyEgg()
{
streamFactory = new StreamFactory();
processProxy = new ProcessProxy();
var pofContext = new PofContext().With(x => {
x.MergeContext(new DspPofContext());
x.MergeContext(new TrinketsApiPofContext());
x.MergeContext(new TrinketsImplPofContext());
});
ICollectionFactory collectionFactory = new CollectionFactory();
IFileSystemProxy fileSystemProxy = new FileSystemProxy(streamFactory);
IThreadingProxy threadingProxy = new ThreadingProxy(new ThreadingFactory(), new SynchronizationFactory());
var dnsProxy = new DnsProxy();
INetworkingProxy networkingProxy = new NetworkingProxy(new SocketFactory(new TcpEndPointFactory(dnsProxy), new NetworkingInternalFactory(threadingProxy, streamFactory)), new TcpEndPointFactory(dnsProxy));
pofSerializer = new PofSerializer(pofContext);
PofStreamsFactory pofStreamsFactory = new PofStreamsFactoryImpl(threadingProxy, streamFactory, pofSerializer);
ProxyGenerator proxyGenerator = new ProxyGenerator();
var serviceClientFactory = new ServiceClientFactoryImpl(proxyGenerator, streamFactory, collectionFactory, threadingProxy, networkingProxy, pofSerializer, pofStreamsFactory);
// construct libdsp local service node
ClusteringConfiguration clusteringConfiguration = new ClientClusteringConfiguration();
ServiceClient localServiceClient = serviceClientFactory.Construct(clusteringConfiguration);
keepaliveObjects.Add(localServiceClient);
temporaryFileService = localServiceClient.GetService<TemporaryFileService>();
var processInjector = new ProcessInjectorImpl();
ProcessInjectionConfiguration processInjectionConfiguration = new ProcessInjectionConfigurationImpl(injectionAttempts: 10, injectionAttemptsDelay: 200);
processInjectionService = new ProcessInjectionServiceImpl(processInjector, processInjectionConfiguration);
IDtpNodeFactory transportNodeFactory = new DefaultDtpNodeFactory();
BootstrapConfigurationGenerator bootstrapConfigurationGenerator = new BootstrapConfigurationGeneratorImpl();
trinketInternalUtilities = new TrinketInternalUtilitiesImpl(fileSystemProxy);
trinketDtpServerFactory = new TrinketDtpServerFactoryImpl(streamFactory, transportNodeFactory, bootstrapConfigurationGenerator);
}
// Snippet Start
public static void Run(string fileName)
{
// DataSet is the object used to interact with the data file.
// StreamFactory creates Dataset with pool of binary readers to
// perform device lookup using file on disk. The type if
// disposable and is therefore contained in using block to
// ensure file handles and resources are freed.
using (DataSet dataSet = StreamFactory.Create(fileName, false))
{
// Provides access to device detection functions.
Provider provider = new Provider(dataSet);
// Used to store and access detection results.
Match match;
// Contains detection result for the IsMobile property.
string IsMobile;
// User-Agent string of an iPhone mobile device.
string mobileUserAgent = ("Mozilla/5.0 (iPhone; CPU iPhone " +
"OS 7_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like " +
"Gecko) 'Version/7.0 Mobile/11D167 Safari/9537.53");
// User-Agent string of Firefox Web browser version 41 on desktop.
string desktopUserAgent = ("Mozilla/5.0 (Windows NT 6.3; " +
"WOW64; rv:41.0) Gecko/20100101 Firefox/41.0");
// User-Agent string of a MediaHub device.
string mediaHubUserAgent = ("Mozilla/5.0 (Linux; Android " +
"4.4.2; X7 Quad Core Build/KOT49H) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 " +
"Safari/537.36");
Console.WriteLine("Staring Getting Started Example.");
// Carries out a match for a mobile User-Agent.
Console.WriteLine("\nMobile User-Agent: " + mobileUserAgent);
match = provider.Match(mobileUserAgent);
IsMobile = match["IsMobile"].ToString();
Console.WriteLine(" IsMobile: " + IsMobile);
// Carries out a match for a desktop User-Agent.
Console.WriteLine("\nDesktop User-Agent: " + desktopUserAgent);
match = provider.Match(desktopUserAgent);
IsMobile = match["IsMobile"].ToString();
Console.WriteLine(" IsMobile: " + IsMobile);
// Carries out a match for a MediaHub User-Agent.
Console.WriteLine("\nMediaHub User-Agent: " + mediaHubUserAgent);
match = provider.Match(mediaHubUserAgent);
IsMobile = match["IsMobile"].ToString();
Console.WriteLine(" IsMobile: " + IsMobile);
// Returns the number of profiles that are Mobile.
Console.WriteLine("\nNumber of mobile profiles: {0}",
dataSet.FindProfiles("IsMobile", "True").Length);
}
}
public IncomingMessageQueue(StreamFactory streamFactory, PipeStream readStream,
NamedPipeCommunicationProtocolAgent communicationProtocol,
CancellationToken cancellationToken)
{
this.streamFactory = streamFactory;
this.readStream = readStream;
this.communicationProtocol = communicationProtocol;
this.cancellationToken = cancellationToken;
}
public NamedPipeInstanceCommunicationService([KeyFilter(CommunicationConstants.MessageSenderContainerKey)] IContainer messageSenderContainer,
ILog log, StreamFactory streamFactory, IEnvironmentInformation environmentInformation, CancellationToken cancellationToken)
{
this.messageSenderContainer = messageSenderContainer;
this.log = log;
this.streamFactory = streamFactory;
this.environmentInformation = environmentInformation;
this.cancellationToken = cancellationToken;
}
private NamedPipeCommunicationProtocolSimulator(PipeStream readStream, PipeStream writeStream,
StreamFactory streamFactory, ILog log)
: base(readStream, writeStream, streamFactory, log)
{
this.readStream = readStream;
this.writeStream = writeStream;
this.streamFactory = streamFactory;
this.log = log;
}
/// <summary>
/// Opens the underlying stream, if it is not set externally via
/// a constructor or the OpenStream method.
/// </summary>
private void OpenOwnStream()
{
// Use stream factory to open stream
var f = StreamFactory.Create(streamFactoryType);
baseStream = f.Open(uri, fileMode);
ownsBaseStream = true;
}
/// <summary>
/// Activates the data array containing the premium data.
/// </summary>
/// <param name="data">Data to activate</param>
public static LicenceKeyResults Activate(byte[] data)
{
try
{
DataSet dataSet = null;
// Validate the data provided is correct.
try
{
dataSet = StreamFactory.Create(data);
}
catch (MobileException ex)
{
EventLog.Warn(ex);
return(LicenceKeyResults.DataInvalid);
}
// Check the configuration.
try
{
CheckConfig();
}
catch (Exception ex)
{
EventLog.Warn(ex);
return(LicenceKeyResults.Config);
}
// Write the file to the binary data path.
try
{
File.WriteAllBytes(Detection.Configuration.Manager.BinaryFilePath, data);
File.SetLastAccessTimeUtc(Detection.Configuration.Manager.BinaryFilePath, dataSet.Published);
}
catch (IOException ex)
{
EventLog.Warn(ex);
return(LicenceKeyResults.WriteDataFile);
}
// Switch in the new data to complete activation.
WebProvider.Refresh();
EventLog.Info(String.Format(
"Activated binary data file '{0}' with new version " +
"dated the '{1:d}'.",
AutoUpdate.MasterBinaryDataFile.FullName,
dataSet.Published));
}
catch (Exception ex)
{
EventLog.Warn(ex);
return(LicenceKeyResults.GenericFailure);
}
return(LicenceKeyResults.Success);
}
public StreamFactory GetStreamFactory()
{
if (instance == null)
{
instance = new FileStreamFactory();
}
return(instance);
}
/// <summary>
/// Ruft die Meldedatei mit einem bestimmten Dateinamen aus dem Deuev-Ordner ab
/// </summary>
/// <param name="fileName">
/// Dateiname der Meldedatei
/// </param>
/// <param name="name">
/// Name in der Meldungen.xml
/// </param>
/// <returns>
/// Meldedatei als DeuevMessageData-Objekt
/// </returns>
private DsraMessageData GetMessageFromFile(string fileName, string name)
{
var input = string.Join(string.Empty, File.ReadAllLines(fileName).Select(x => x + Environment.NewLine));
var output = new StringWriter();
var writer = StreamFactory.CreateWriter(name, output);
var reader = StreamFactory.CreateReader(name, new StringReader(input));
var deuevMessage = new DsraMessageData();
try
{
var streamObject = reader.Read();
do
{
var vosz = Assert.IsType <VOSZ>(streamObject);
deuevMessage.VOSZ.Add(vosz);
writer.Write(vosz);
streamObject = reader.Read();
}while (reader.RecordName == "VOSZ" || reader.RecordName == "VOSZ-v01");
if (reader.RecordName == "DSKO" || reader.RecordName == "DSKO-v04")
{
var dsko = Assert.IsType <DSKO04>(streamObject);
writer.Write(dsko);
streamObject = reader.Read();
}
while (reader.RecordName == "DSRA" || reader.RecordName == "DSRA-v01")
{
var record = Assert.IsType <DSRA01>(streamObject);
deuevMessage.DSRA01.Add(record);
writer.Write(record);
streamObject = reader.Read();
}
do
{
var ncsz = Assert.IsType <NCSZ>(streamObject);
writer.Write(streamObject);
deuevMessage.NCSZ.Add(ncsz);
streamObject = reader.Read();
}while (reader.RecordName != null && (reader.RecordName == "NCSZ" || reader.RecordName == "NCSZ-v01"));
Assert.Null(reader.RecordName);
Assert.Equal(deuevMessage.VOSZ.Count, deuevMessage.NCSZ.Count);
writer.Close();
string output2 = output.ToString();
Assert.Equal(input, output2);
return(deuevMessage);
}
finally
{
reader.Close();
}
}
public void GenerateOutputFile()
{
SetFullOutputFileName();
using var streamWriter = StreamFactory.CreateStreamWriter(FullOutputFileName);
WriteDates(streamWriter);
WriteColumnHeadings(streamWriter);
WriteRows(streamWriter);
}
/// <summary>
/// This method is called when Sample load is requested, if you return a Sample
/// it will return it from the user else it will keep trying all the other SampleLoaders
/// until it does get one
/// </summary>
/// <param name="path">File path of the sound to load.</param>
/// <param name="streamed">If set to true the sound will be loaded in piece by piece rather than all at once.</param>
/// <returns>New Sample instance or NULL if this factory can't load the given audio sample file format.</returns>
protected override Sample RequestLoad(object path, bool streamed)
{
// Load in the audio file's data.
Stream stream = StreamFactory.RequestStream(path, StreamMode.Open);
if (stream == null)
{
return(null);
}
if (stream.ReadByte() != 'O' || stream.ReadByte() != 'g' || stream.ReadByte() != 'g' || stream.ReadByte() != 'S')
{
stream.Close();
return(null);
}
stream.Position = 0;
byte[] data = new byte[stream.Length];
stream.Read(data, 0, (int)stream.Length);
stream.Close();
// Create an audiere memory file to place our data into.
ManagedAudiere.File file = Audiere.CreateMemoryFile(new MemoryFileBuffer(data, data.Length));
SampleSource audiereSample = Audiere.OpenSampleSource(file);
SampleFormatData sampleFormat = audiereSample.GetFormat();
int size = audiereSample.Length * (Audiere.GetSampleSize(sampleFormat.sample_format) * sampleFormat.channel_count);
MemoryFileBuffer audiereBuffer = new MemoryFileBuffer(new byte[size], size);
audiereSample.Read(audiereSample.Length, audiereBuffer);
// Put the audiere audio buffer into a new sample buffer.
byte[] pcmBuffer = audiereBuffer.GetBuffer();
Sample sample = null;
if (sampleFormat.channel_count == 1)
{
sample = new Sample(SampleFormat.MONO16LE, pcmBuffer.Length);
}
else
{
sample = new Sample(SampleFormat.STEREO16LE, pcmBuffer.Length);
}
sample.SampleRate = sampleFormat.sample_rate;
sample.ChannelCount = sampleFormat.channel_count;
sample.BlockAlign = Audiere.GetSampleSize(sampleFormat.sample_format) * sampleFormat.channel_count;
sample.ByteRate = sample.SampleRate * sample.BlockAlign;
sample.BitsPerSample = Audiere.GetSampleSize(sampleFormat.sample_format) * 8;
for (int i = 0; i < pcmBuffer.Length; i++)
{
sample.Data[i] = pcmBuffer[i];
}
return(sample);
}
public void ReactiveResourceConverter_StreamFactory_Roundtrip_Parameterized_StreamFactory()
{
var streamFactory = new StreamFactory(new Uri("test://uri"), Expression.Default(typeof(IAsyncReactiveQubjectFactory <int, int, int>)), null);
var serializer = new SerializationHelpers(new DataConverter[] { new ReactiveResourceConverter() });
var serialized = serializer.Serialize(streamFactory);
var roundtripped = serializer.Deserialize <IAsyncReactiveStreamFactoryDefinition>(serialized);
AssertEqual(streamFactory, roundtripped);
Assert.IsTrue(roundtripped.IsParameterized);
}
public void WhenStreamIsMemoryStream_ThenReturnBytes()
{
var sut = StreamFactory.Create(new byte[] { 0xA1, 0xA2 });
var result = sut.ReadAsBytes();
Assert.That(result.Length, Is.EqualTo(2));
Assert.That(result.First(), Is.EqualTo(0xA1));
Assert.That(result.Second(), Is.EqualTo(0xA2));
}
public void WhenStreamIsNotNull_ThenReturnMemoryStream()
{
var sut = StreamFactory.Create("ABC");
var result = sut.ReadAsMemoryStream();
var text = result.ReadAsString();
Assert.That(text, Is.EqualTo("ABC"));
}
/// <summary>
/// De serializes the specified stream.
/// </summary>
/// <param name="stream">The stream.</param>
/// <returns>The object which deserialize</returns>
public object Deserialize(Stream stream)
{
//Creates internal object model from a stream, which Interpreter will analyze into an object
BaseDescriptor descriptor = StreamFactory.CreateReader().Read(stream);
var interperter = new Interpreter();
var result = interperter.Analyze(descriptor);
return(result);
}
public async Task WhenStreamContainsText_ThenWriteToFile()
{
var filePath = Path.Combine(WorkingDir, "Stream-WriteToFile1.txt");
var sut = StreamFactory.Create("ABC");
await sut.WriteToFileAsync(filePath);
AssertFile.SizeEquals(filePath, 3);
}
public List <RetVal> PredictOccupancy(long startSlotIndex, long endSlotIndex)
{
long average = 0;
List <RetVal> retVal = new List <RetVal>();
StreamFactory streamFactory = StreamFactory.Instance;
FqStreamID fq_sid = new FqStreamID(fqsidprefix + chunkSize, "A", "TestBS");
CallerInfo ci = new CallerInfo(null, "A", "A", 1);
Directory.Delete(fq_sid.HomeId, true);
int slotIndex = Convert.ToInt32(startSlotIndex); long startTime = 0, retrievelTime = 0, computeTime = 0;
IStream occupancyGroundTruthStream = streamFactory.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Read, mdserver, chunkSize, 1, new Logger());
while (true)
{
List <int> currentPOV = new List <int>();
List <List <int> > previousDaysPOV = new List <List <int> >();
try
{
startTime = DateTime.Now.Ticks;
currentPOV = ConstructCurrentPOV(occupancyGroundTruthStream, slotIndex);
previousDaysPOV = ConstructPreviousPOV(occupancyGroundTruthStream, slotIndex);
retrievelTime = DateTime.Now.Ticks - startTime;
startTime = DateTime.Now.Ticks;
int predictedOccupancy = Predict(currentPOV, previousDaysPOV);
computeTime = DateTime.Now.Ticks - startTime;
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Console.WriteLine("Slot number {0} {1} ", slotIndex, retrievelTime);
using (results = File.AppendText(outputFilePath))
results.WriteLine("Slot number {0} {1}", slotIndex, retrievelTime);
average += retrievelTime;
slotIndex++;
if (slotIndex == endSlotIndex)
{
break;
}
}
occupancyGroundTruthStream.Close();
average = average / (endSlotIndex - startSlotIndex + 1);
retVal.Add(new RetVal(0, Convert.ToInt32(average)));
return(retVal);
}
public IncomingMessageQueue(StreamFactory streamFactory, ILog log, PipeStream readStream,
NamedPipeCommunicationProtocol communicationProtocol,
OutgoingMessageQueue outgoingMessageQueue, CancellationToken cancellationToken)
{
this.streamFactory = streamFactory;
this.log = log;
this.readStream = readStream;
this.communicationProtocol = communicationProtocol;
this.cancellationToken = cancellationToken;
this.outgoingMessageQueue = outgoingMessageQueue;
}
private static void UploadDataAsStreams(int UploadCount)
{
string directory = @"..\\..\\data\\meter-data";
int count = 0;
Dictionary <DateTime, double> ts_temperature = new Dictionary <DateTime, double>();
StreamReader wfile = new System.IO.StreamReader(@"..\\..\\data\\weather.txt");
string wline;
while ((wline = wfile.ReadLine()) != null)
{
string[] words = wline.Split('\t');
DateTime date = Convert.ToDateTime(words[4]);
date = date.AddHours(Int32.Parse(words[5]));
double temperature = Double.Parse(words[0]);
ts_temperature[date] = temperature;
}
wfile.Close();
foreach (string filePath in Directory.GetFiles(directory))
{
Console.WriteLine("file name:" + filePath);
string line;
System.IO.StreamReader file = new System.IO.StreamReader(filePath);
LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
FqStreamID fq_sid = new FqStreamID("crrealhome" + count, "A", "TestBS");
CallerInfo ci = new CallerInfo(null, "A", "A", 1);
StreamFactory sf = StreamFactory.Instance;
sf.deleteStream(fq_sid, ci);
IStream dfs_byte_val = sf.openValueDataStream <DoubleKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write, mdserveraddress: mdServer, ChunkSizeForUpload: 4 * 1024 * 1024, ThreadPoolSize: 1, log: new Logger());
while ((line = file.ReadLine()) != null)
{
string[] words = line.Split('\t');
DateTime date = Convert.ToDateTime(words[0]);
date = date.AddHours(int.Parse(words[1]) / 100);
DoubleKey key = new DoubleKey(((int)(ts_temperature[date])));
dfs_byte_val.Append(key, new ByteValue(BitConverter.GetBytes(Double.Parse(words[2]))), DateTimeToUnixTimestamp(date));
// Console.WriteLine(DateTimeToUnixTimestamp(date) + "," + words[2]);
}
dfs_byte_val.Close();
count++;
if (count == UploadCount)
{
break;
}
}
}
public InterProcessUpdateReceiver(IEnvironmentInformation environmentInformation, StreamFactory streamFactory, ILog log, ICommunicationProtocol externalProtocol, IMessageParser messageParser)
{
serverName = environmentInformation.InterProcessServerNameBase +
environmentInformation.CurrentProcessId.ToString("D", CultureInfo.InvariantCulture);
this.streamFactory = streamFactory;
this.log = log;
this.externalProtocol = externalProtocol;
this.messageParser = messageParser;
messageParser.HandshakeCompleted += MessageParserOnHandshakeCompleted;
StartAndWaitForInterProcessUpdate();
}
public void OpenStream(ScriptThread thread)
{
Stream stream = StreamFactory.RequestStream(thread.GetStringParameter(0), (StreamMode)thread.GetIntegerParameter(1));
if (stream == null)
{
DebugLogger.WriteLog((thread.Process.Url != null && thread.Process.Url != "" ? thread.Process.Url : "A script") + " called OpenStream with an unreachable url.", LogAlertLevel.Error);
return;
}
thread.SetReturnValue(new StreamScriptObject(new ScriptStream(stream)));
}
public void WhenStreamContainsText_ThenReturnBase64()
{
const string text = "John Smith is my name.";
var sut = StreamFactory.Create(text);
var result = sut.ReadAsBase64();
Assert.That(result, Is.EqualTo("Sm9obiBTbWl0aCBpcyBteSBuYW1lLg=="));
Assert.That(new Base64Encoder().Decode(result), Is.EqualTo(text));
}
/// <summary>
/// Ruft die Meldedatei mit einem bestimmten Dateinamen aus dem Deuev-Ordner ab
/// </summary>
/// <param name="input">Die Meldedatei</param>
/// <param name="name">Name in der Meldungen.xml</param>
/// <returns>Meldedatei als DeuevMessageData-Objekt</returns>
private BwnaMessageData GetMessageFromString(string input, string name)
{
var output = new StringWriter();
var writer = StreamFactory.CreateWriter(name, output);
var reader = StreamFactory.CreateReader(name, new StringReader(input));
var deuevMessage = new BwnaMessageData();
try
{
var streamObject = reader.Read();
do
{
var vosz = Assert.IsType <VOSZ>(streamObject);
deuevMessage.VOSZ.Add(vosz);
writer.Write(vosz);
streamObject = reader.Read();
}while (reader.RecordName == "VOSZ" || reader.RecordName == "VOSZ-v01");
var dsko = Assert.IsType <DSKO02>(streamObject);
deuevMessage.DSKO = dsko;
writer.Write(dsko);
streamObject = reader.Read();
while (reader.RecordName == "DSBE" || reader.RecordName == "DSBE-v01")
{
var record = Assert.IsType <DSBE01>(streamObject);
deuevMessage.DSBEv01.Add(record);
writer.Write(record);
streamObject = reader.Read();
}
do
{
var ncsz = Assert.IsType <NCSZ>(streamObject);
writer.Write(streamObject);
deuevMessage.NCSZ.Add(ncsz);
streamObject = reader.Read();
}while (reader.RecordName != null && (reader.RecordName == "NCSZ" || reader.RecordName == "NCSZ-v01"));
Assert.Null(reader.RecordName);
Assert.Equal(deuevMessage.VOSZ.Count, deuevMessage.NCSZ.Count);
writer.Close();
string output2 = output.ToString();
Assert.Equal(input, output2);
return(deuevMessage);
}
finally
{
reader.Close();
}
}
public void Setup()
{
k1 = new StrKey("k1");
k2 = new StrKey("k2");
string AzureaccountName = "msrlot";
string AzureaccountKey = "wC3ou+VLBTu8ryKbsiSMtsIIHIpxGSIAWA0NtK58da2wELQ+USgYQwmVMtyq/p8ILsuZc8TiLeHBjaPI+a3A2Q==";
locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
streamID = new FqStreamID("99-a2000", "A0", "TestDS");
callerInfo = new CallerInfo(null, "A0", "A0", 1);
streamSecurityType = StreamFactory.StreamSecurityType.Plain;
streamFactory = StreamFactory.Instance;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="factory">Factory method to create underlying stream.</param>
public SuspendResumeStream(StreamFactory factory)
{
this.syncObject = new object();
this.stateLock = new object();
this.state = State.Initial;
this.factory = factory;
this.powerModeChangedEventHandler = new Microsoft.Win32.PowerModeChangedEventHandler(SystemEvents_PowerModeChanged);
Microsoft.Win32.SystemEvents.PowerModeChanged += this.powerModeChangedEventHandler;
this.TryStateTransition(State.Active, this.Open);
}
public async Task WhenFileAlreadyExists_ThenOverWrite()
{
var filePath = Path.Combine(WorkingDir, "Stream-WriteToFile2.txt");
FileBuilder.InFileSystem.WithPath(filePath).WithText("A").Build();
var sut = StreamFactory.Create("ABC");
await sut.WriteToFileAsync(filePath);
AssertFile.SizeEquals(filePath, 3);
}
// Snippet Start
public static void Run(string fileName)
{
// DataSet is the object used to interact with the data file.
// StreamFactory creates Dataset with pool of binary readers to
// perform device lookup using file on disk.
DataSet dataSet = StreamFactory.Create(fileName, false);
// Name of the output file results will be saved to.
string outputFile = "AllProfilesOutput.csv";
// Get all the hardware profiles from the hardware component.
Profile[] hardwareProfiles = dataSet.Hardware.Profiles;
// Get all the hardware properties from the hardware component.
// For the full list of properties see:
// https://51degrees.com/resources/property-dictionary
Property[] hardwareProperties = dataSet.Hardware.Properties;
// Opens and output file.
StreamWriter fout = new StreamWriter(outputFile);
Console.WriteLine("Starting All Profiles Example.");
// Print CSV headers to output file.
fout.Write("Id");
foreach (Property property in hardwareProperties)
{
fout.Write("," + property.Name);
}
fout.Write("\n");
// Loop over all devices.
foreach (Profile profile in hardwareProfiles)
{
// Write the device's profile id.
fout.Write(profile.ProfileId);
foreach (Property property in hardwareProperties)
{
// Print the profile's values for the property.
fout.Write(",");
fout.Write(getProfileValues(profile, property));
}
fout.Write("\n");
}
fout.Close();
Console.WriteLine("Output Written to " + outputFile);
// Finally close the dataset, releasing resources and file locks.
dataSet.Dispose();
}
private static void StreamLoggerTest(LoggerFactory loggerFactory)
{
using (var stream = new StreamFactory().Create())
{
var logger = loggerFactory.CreateStreamLogger(stream, new StreamWriter());
logger.Log(LogLevel.Debug, LoggerProvider.CreateLogMessage(MessageLength.Normal));
logger.Log(LogLevel.Info, LoggerProvider.CreateLogMessage(MessageLength.Normal));
logger.Log(LogLevel.Error, LoggerProvider.CreateLogMessage(MessageLength.Normal));
var content = $"From Stream: {LoggerProvider.GetStreamContent(stream)}";
System.Console.WriteLine(content);
}
}
public void Setup()
{
k1 = new StrKey("k1");
k2 = new StrKey("k2");
string AzureaccountName = "testdrive";
string AzureaccountKey = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q==";
locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
streamID = new FqStreamID("99-a2000", "A0", "TestDS");
callerInfo = new CallerInfo(null, "A0", "A0", 1);
streamSecurityType = StreamFactory.StreamSecurityType.Plain;
streamFactory = StreamFactory.Instance;
}
private void saveMixtureWeightsAscii(GaussianWeights gaussianWeights, string text, bool append)
{
this.logger.info("Saving mixture weights to: ");
this.logger.info(text);
OutputStream outputStream = StreamFactory.getOutputStream(this.location, text, append);
if (outputStream == null)
{
string text2 = new StringBuilder().append("Error trying to write file ").append(this.location).append(text).toString();
throw new IOException(text2);
}
PrintWriter printWriter = new PrintWriter(outputStream, true);
printWriter.print("mixw ");
int statesNum = gaussianWeights.getStatesNum();
printWriter.print(new StringBuilder().append(statesNum).append(" ").toString());
int streamsNum = gaussianWeights.getStreamsNum();
printWriter.print(new StringBuilder().append(streamsNum).append(" ").toString());
int gauPerState = gaussianWeights.getGauPerState();
printWriter.println(gauPerState);
for (int i = 0; i < statesNum; i++)
{
for (int j = 0; j < streamsNum; j++)
{
printWriter.print(new StringBuilder().append("mixw [").append(i).append(" ").append(j).append("] ").toString());
float[] array = new float[gauPerState];
float[] array2 = new float[gauPerState];
for (int k = 0; k < gauPerState; k++)
{
array2[k] = gaussianWeights.get(i, j, k);
}
this.logMath.logToLinear(array2, array);
float num = 0f;
for (int l = 0; l < gauPerState; l++)
{
num += array[l];
}
printWriter.println(num);
printWriter.print("\n\t");
for (int l = 0; l < gauPerState; l++)
{
printWriter.print(new StringBuilder().append(" ").append(array[l]).toString());
}
printWriter.println();
}
}
outputStream.close();
}
private static long RemoteRead(int numberOfHomes, DateTime start, DateTime end, string tag)
{
Dictionary <int, List <double> > temp_energy_allhomes = new Dictionary <int, List <double> >();
Dictionary <int, List <double> > temp_energy_home;
long retVal = 0;
for (int i = 0; i < numberOfHomes; i++)
{
temp_energy_home = new Dictionary <int, List <double> >();
LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
FqStreamID fq_sid = new FqStreamID("crrealhome" + i, "A", "TestBS");
CallerInfo ci = new CallerInfo(null, "A", "A", 1);
StreamFactory sf = StreamFactory.Instance;
IStream dfs_byte_val = sf.openValueDataStream <DoubleKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Read, mdServer, 4 * 1024 * 1024, 1, new Logger());
long start_ticks = DateTime.Now.Ticks;
for (int temp = -30; temp <= 40; temp++)
{
IEnumerable <IDataItem> vals = dfs_byte_val.GetAll(new DoubleKey(temp), DateTimeToUnixTimestamp(start), DateTimeToUnixTimestamp(end));
if (vals != null)
{
foreach (IDataItem val in vals)
{
if (!temp_energy_home.ContainsKey(temp))
{
temp_energy_home[temp] = new List <double>();
}
if (!temp_energy_allhomes.ContainsKey(temp))
{
temp_energy_allhomes[temp] = new List <double>();
}
temp_energy_home[temp].Add(BitConverter.ToDouble(val.GetVal().GetBytes(), 0));
temp_energy_allhomes[temp].Add(BitConverter.ToDouble(val.GetVal().GetBytes(), 0));
}
}
}
dfs_byte_val.Close();
long end_ticks = DateTime.Now.Ticks;
retVal += end_ticks - start_ticks;
WriteToFile(".\\result-realhome-" + i + "-n-" + numberOfHomes + "-" + tag, temp_energy_home);
}
WriteToFile(".\\result-allhomes-n-" + numberOfHomes + "-" + tag, temp_energy_allhomes);
return(retVal);
}
public void LoadScreen()
{
PeaksTitle.Text = LabelHandler.Instance.GetPeaksTitle(MenuSelection);
UnitsLabel1.Text = LabelHandler.Instance.GetPeaksLabel(MenuSelection);
UnitsLabel2.Text = LabelHandler.Instance.GetPeaksLabel(MenuSelection);
UnitsLabel3.Text = LabelHandler.Instance.GetPeaksLabel(MenuSelection);
UnitsLabel4.Text = LabelHandler.Instance.GetPeaksLabel(MenuSelection);
DataManipulatorHandler.CreateSingleton();
Stream = StreamFactory.GetSingleton(Activity, AccessToken).CreateStream(MenuSelection);
LoadChart(Stream);
LoadLabels(Stream);
LoadCommands();
}
public static void Demo()
{
StreamFactory factory = GenerateSampleData;
using (Stream stream = factory())
{
int data;
while ((data = stream.ReadByte()) != -1)
{
Console.WriteLine(data);
}
}
}
public ThumbnailGenerationEgg()
{
collectionFactory = new CollectionFactory();
var streamFactory = new StreamFactory();
var processProxy = new ProcessProxy();
threadingProxy = new ThreadingProxy(new ThreadingFactory(), new SynchronizationFactory());
pofContext = new PofContext().With(x => {
x.MergeContext(new ManagementPofContext());
x.MergeContext(new ThumbnailGeneratorApiPofContext());
});
pofSerializer = new PofSerializer(pofContext);
networkingProxy = new NetworkingProxy(new SocketFactory(new TcpEndPointFactory(new DnsProxy()), new NetworkingInternalFactory(threadingProxy, streamFactory)), new TcpEndPointFactory(new DnsProxy()));
thumbnailGeneratorService = new ThumbnailGeneratorServiceImpl();
}
public static int Main()
{
ICollectionFactory collectionFactory = new CollectionFactory();
ProxyGenerator proxyGenerator = new ProxyGenerator();
IThreadingFactory threadingFactory = new ThreadingFactory();
ISynchronizationFactory synchronizationFactory = new SynchronizationFactory();
IThreadingProxy threadingProxy = new ThreadingProxy(threadingFactory, synchronizationFactory);
IDnsProxy dnsProxy = new DnsProxy();
ITcpEndPointFactory tcpEndPointFactory = new TcpEndPointFactory(dnsProxy);
IStreamFactory streamFactory = new StreamFactory();
INetworkingInternalFactory networkingInternalFactory = new NetworkingInternalFactory(threadingProxy, streamFactory);
ISocketFactory socketFactory = new SocketFactory(tcpEndPointFactory, networkingInternalFactory);
INetworkingProxy networkingProxy = new NetworkingProxy(socketFactory, tcpEndPointFactory);
IPofContext pofContext = new ClientPofContext();
IPofSerializer pofSerializer = new PofSerializer(pofContext);
PofStreamsFactory pofStreamsFactory = new PofStreamsFactoryImpl(threadingProxy, streamFactory, pofSerializer);
var serviceConfiguration = new ClientClusteringConfiguration();
var serviceClientFactory = new ServiceClientFactoryImpl(proxyGenerator, streamFactory, collectionFactory, threadingProxy, networkingProxy, pofSerializer, pofStreamsFactory);
var localEndPoint = tcpEndPointFactory.CreateLoopbackEndPoint(serviceConfiguration.Port);
var reconnectAttempts = 10;
var reconnectDelay = 1000;
var serviceClient = TryConnectToEndpoint(reconnectAttempts, reconnectDelay, serviceClientFactory, serviceConfiguration);
if (serviceClient == null) {
Console.Error.WriteLine("Failed to connect to endpoint.");
return 1;
} else {
var dispatcher = new DispatcherCommand("registered commands");
dispatcher.RegisterCommand(new ShutdownCommand(serviceClient));
// dispatcher.RegisterCommand(new ModCommand(serviceClient));
dispatcher.RegisterCommand(new ExitCommand());
dispatcher.RegisterCommand(new ServiceCommand(serviceClient));
var repl = new DargonREPL(dispatcher);
repl.Run();
return 0;
}
}
private static int Main(string[] args)
{
var options = new Options();
try
{
if (!CommandLine.Parser.Default.ParseArguments(args, options))
{
var usage = options.GetUsage();
Console.WriteLine(usage);
return -1;
}
var factory = new StreamFactory(options);
var cleaner = new Cleaner(factory);
cleaner.Clean();
}
catch (Exception ex)
{
Console.Error.WriteLine("Error formatting JSON");
if (options.Verbose)
{
Console.Error.WriteLine(ex);
}
else
{
Console.Error.WriteLine(ex.Message);
}
return -1;
}
return 0;
}
public ListTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
ObjectInspector childOI = inspector == null ? null :
((ListObjectInspector)inspector).getListElementObjectInspector();
childrenWriters = new TreeWriter[1];
childrenWriters[0] =
createTreeWriter(childOI, schema.getChildren()[0], writer, true);
lengths = createIntegerWriter(writer.createStream(columnId,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public MapTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
ObjectInspector keyInspector = null;
ObjectInspector valueInspector = null;
if (inspector != null)
{
MapObjectInspector insp = (MapObjectInspector)inspector;
keyInspector = insp.getMapKeyObjectInspector();
valueInspector = insp.getMapValueObjectInspector();
}
childrenWriters = new TreeWriter[2];
IList<TypeDescription> children = schema.getChildren();
childrenWriters[0] = createTreeWriter(keyInspector, children[0], writer, true);
childrenWriters[1] = createTreeWriter(valueInspector, children[1], writer, true);
lengths = createIntegerWriter(writer.createStream(columnId,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public VarcharTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
maxLength = schema.getMaxLength();
}
public Cleaner(StreamFactory streamFactory)
{
_stramFactory = streamFactory;
}
public DateTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
OutStream @out = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.isDirectV2 = isNewWriteFormat(writer);
this.writer = createIntegerWriter(@out, true, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public static void Main(string[] args)
{
Console.WindowWidth = Console.BufferWidth = 120;
var streamFactory = new StreamFactory();
IFileSystemProxy fileSystemProxy = new FileSystemProxy(streamFactory);
DargonNodeFactory dargonNodeFactory = new DargonNodeFactoryImpl();
var dispatcher = new DispatcherCommand("root");
dispatcher.RegisterCommand(new ExitCommand());
dispatcher.RegisterCommand(new LoadSolutionCommand(streamFactory, dargonNodeFactory));
dispatcher.RegisterCommand(new ChangeDirectoryCommand());
dispatcher.RegisterCommand(new ListDirectoryCommand());
dispatcher.RegisterCommand(new PrintWorkingDirectoryCommand());
dispatcher.RegisterCommand(new AliasCommand("dir", new ListDirectoryCommand()));
dispatcher.RegisterCommand(new DumpCommand(fileSystemProxy));
dispatcher.RegisterCommand(new SetWindowWidthCommand());
dispatcher.Eval(LoadSolutionCommand.kLoadSolutionCommand + " " + (ConfigurationManager.AppSettings?["radsPath"] ?? ""));
new ReplCore(dispatcher).Run();
}
public List<string> Dump(string CallerName,
string HomeName, string AppName, string StreamName,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamPhysicalType ptype,
Byte[] value, int num_operations,
SynchronizerType synctype,
bool doCosts= false)
{
List<string> exp_details = new List<string>();
exp_details.Add(exp_id);
File.AppendAllLines(exp_directory + "/exp", exp_details);
return exp_details;
}
public static string doDiskSpeed(string filesize, string blocksize, StreamFactory.StreamOp op)
{
// For the example
// Use ProcessStartInfo class
try
{
File.Delete("testfile.dat");
}
catch
{
}
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.CreateNoWindow = false;
startInfo.UseShellExecute = false;
startInfo.FileName = "DiskSpd.exe";
startInfo.WindowStyle = ProcessWindowStyle.Hidden;
startInfo.Arguments = " -UN " ;
if (op == StreamFactory.StreamOp.Write)
startInfo.Arguments += " -w ";
startInfo.Arguments += " -b" + blocksize;
startInfo.Arguments += " -c" + filesize;
startInfo.Arguments += " -d5";
startInfo.Arguments += " testfile.dat";
startInfo.RedirectStandardOutput = true;
string ret = "";
try
{
// Start the process with the info we specified.
// Call WaitForExit and then the using statement will close.
using (Process exeProcess = Process.Start(startInfo))
{
ret = exeProcess.StandardOutput.ReadToEnd();
exeProcess.WaitForExit();
}
}
catch (Exception e)
{
Console.WriteLine("Exception {0}", e);
// Log error.
}
return ret;
}
public StringTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
}
public StructTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
IList<TypeDescription> children = schema.getChildren();
if (inspector != null)
{
StructObjectInspector structObjectInspector =
(StructObjectInspector)inspector;
fields = structObjectInspector.getAllStructFieldRefs();
}
childrenWriters = new TreeWriter[children.Count];
for (int i = 0; i < childrenWriters.Length; ++i)
{
ObjectInspector childOI;
if (fields != null && i < fields.Count)
{
childOI = fields[i].getFieldObjectInspector();
}
else
{
childOI = null;
}
childrenWriters[i] = createTreeWriter(
childOI, children[i], writer,
true);
}
recordPosition(rowIndexPosition);
}
/**
* Create a tree writer.
* @param columnId the column id of the column to write
* @param inspector the object inspector to use
* @param schema the row schema
* @param streamFactory limited access to the Writer's data.
* @param nullable can the value be null?
* @
*/
protected TreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory streamFactory,
bool nullable)
{
this.streamFactory = streamFactory;
this.isCompressed = streamFactory.isCompressed();
this.id = columnId;
this.inspector = inspector;
if (nullable)
{
isPresentOutStream = streamFactory.createStream(id,
OrcProto.Stream.Types.Kind.PRESENT);
isPresent = new BitFieldWriter(isPresentOutStream, 1);
}
else
{
isPresent = null;
}
this.foundNulls = false;
createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
indexStatistics = ColumnStatisticsImpl.create(schema);
stripeColStatistics = ColumnStatisticsImpl.create(schema);
fileStatistics = ColumnStatisticsImpl.create(schema);
childrenWriters = new TreeWriter[0];
rowIndex = OrcProto.RowIndex.CreateBuilder();
rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder();
rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>();
if (streamFactory.buildIndex())
{
rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX);
}
else
{
rowIndexStream = null;
}
if (createBloomFilter)
{
bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder();
bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder();
bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER);
bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP());
}
else
{
bloomFilterEntry = null;
bloomFilterIndex = null;
bloomFilterStream = null;
bloomFilter = null;
}
}
public IntegerWriter createIntegerWriter(PositionedOutputStream output,
bool signed, bool isDirectV2,
StreamFactory writer)
{
if (isDirectV2)
{
bool alignedBitpacking = false;
if (writer.getEncodingStrategy() == OrcFile.EncodingStrategy.SPEED)
{
alignedBitpacking = true;
}
return new RunLengthIntegerWriterV2(output, signed, alignedBitpacking);
}
else
{
return new RunLengthIntegerWriter(output, signed);
}
}
public bool isNewWriteFormat(StreamFactory writer)
{
return writer.getVersion() != OrcFile.Version.V_0_11;
}
public IntegerTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
OutStream @out = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.isDirectV2 = isNewWriteFormat(writer);
this.writer = createIntegerWriter(@out, true, isDirectV2, writer);
if (inspector is IntObjectInspector)
{
intInspector = (IntObjectInspector)inspector;
shortInspector = null;
longInspector = null;
}
else
{
intInspector = null;
if (inspector is LongObjectInspector)
{
longInspector = (LongObjectInspector)inspector;
shortInspector = null;
}
else
{
shortInspector = (ShortObjectInspector)inspector;
longInspector = null;
}
}
recordPosition(rowIndexPosition);
}
public FloatTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.stream = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.utils = new SerializationUtils();
recordPosition(rowIndexPosition);
}
public void Run(string CallerName,
string HomeName, string AppName, string StreamName,
string RandName,
long stime, long etime,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamPhysicalType ptype,
CompressionType ctype, int ChunkSize , int ThreadPoolSize,
Byte[] value, int num_operations,
SynchronizerType synctype,
int max_key = 0,
string address = null,
bool doCosts= false,
bool doRaw = false)
{
// Set experiment directory
CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1);
exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;
if (max_key == 0)
max_key = num_operations;
// Set a description/tag for the experiment
this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName +
" Caller:" + CallerName
+ " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype
+ " Value size: " + value.Length
+ " num_operations: " + max_key
+ " actual_num_ops: " + num_operations
+ " Sync type: " + synctype
+ " Do costs? " + doCosts + "Chunk Size: " + ChunkSize+ " ThreadPool Size:" +ThreadPoolSize;
this.compressed_exp_id =
" ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype
+ " VS: " + value.Length
+ " I:" + num_operations
+ " MK:" + max_key
+ " SYNC: " + synctype+ " chsize: "+ChunkSize + " nThreads: "+ThreadPoolSize ;
// Set remote storage server account info
string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName");
string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey");
LocationInfo Li;
if (synctype == SynchronizerType.Azure)
Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
else if (synctype == SynchronizerType.AmazonS3)
Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3);
else
Li = null;
StreamFactory sf = StreamFactory.Instance;
IStream stream = null;
FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName);
// Set op : R/W
StreamFactory.StreamOp rw;
if (stream_op == StreamOperation.RandomKeyRandomValueAppend
|| stream_op == StreamOperation.RandomKeySameValueAppend
|| stream_op == StreamOperation.SameKeyRandomValueAppend
|| stream_op == StreamOperation.SameKeySameValueAppend)
{
rw = StreamFactory.StreamOp.Write;
}
else
{
rw = StreamFactory.StreamOp.Read;
}
// Initialize costs
CostsHelper costhelper = null;
double baselineStorageKV = 0;
if (doCosts)
{
costhelper = new CostsHelper();
costhelper.getCurrentCpuUsage();
costhelper.getNetworkUsage();
}
if (stream_type == StreamType.CloudRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = new Logger();
Byte[] val = new Byte[value.Length * num_operations];
// DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
// string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd"));
// string bname = String.Format("{0}", Date.ToString("HH-mm-ss"));
// string cname = String.Format("cloudraw-{0}", RandomString(4));
// string bname = String.Format("{0}", RandomString(4));
string cname = String.Format("cloudraw-{0}", RandName);
string bname = String.Format("{0}", RandName);
if (stream_op == StreamOperation.RandomKeyGet ||
stream_op == StreamOperation.RandomKeyGetMultipleSegments ||
stream_op == StreamOperation.RandomKeyGetAll)
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
else
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
return;
}
if (stream_type == StreamType.DiskRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory);
logger.Dump(exp_directory + "/log");
return;
}
// Are we getting raw disk throughput?
if (stream_type == StreamType.Raw)
{
string ret = doDiskSpeed((value.Length * num_operations)/1000 + "K", value.Length/1000 + "K", rw);
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
File.WriteAllText(exp_directory + "/log", ret);
return;
}
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[max_key];
for (int i = 0; i < max_key; ++i)
{
keys[i] = new StrKey("" + i);
}
/*
List<ByteValue> vals = new List<ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
keys[i] = new StrKey("" + i);
vals.Add(new ByteValue(tmp[i]));
// vals[i] = new ByteValue(tmp);
}
*/
Logger log = new Logger();
// Open stream for different types of experiments
if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else
{
return;
}
if (stream_op == StreamOperation.RandomKeyRandomValueAppend)
{
List<ByteValue> vals = new List<ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
vals.Add(new ByteValue(tmp[i]));
}
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += vals[i].Size();
stream.Append(keys[i], vals[i]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeySameValueAppend)
{
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += value.Length;
stream.Append(keys[i], singlebv);
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeySameValueAppend)
{
StrKey key = new StrKey("ExpKey");
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
stream.Append(key, singlebv);
// System.Threading.Thread.Sleep(10);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments)
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(keys[random.Next(0, max_key)]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGetAll)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; )
{
long st = 0;
long et = -1;
Console.WriteLine(stime + ":" + etime);
while (et < st)
{
st = RandomLong(stime, etime, random);
// et = RandomLong(stime, etime, random);
et = st + (10 * 10 * TimeSpan.TicksPerMillisecond);
}
Console.WriteLine(st + ":" + et);
IEnumerable<IDataItem> iterator = stream.GetAll(key, st, et);
foreach (IDataItem data in iterator)
{
data.GetVal();
++i;
if (i == num_operations)
break;
}
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeyRandomValueAppend)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += key.Size();
// baselineStorageKV += vals[i].Size();
// stream.Append(key, vals[i]);
}
stream.Close();
}
else
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(new StrKey("" + random.Next(0,num_operations - 1)));
}
stream.Close();
}
// Dump the instrumentation logs
stream.DumpLogs(exp_directory + "/log");
// Collect costs usage
List<string> costs = new List<string>();
if (doCosts)
{
costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage());
costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage());
costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:true)/1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:false)/1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV/1000.0f);
}
File.AppendAllLines(exp_directory + "/log", costs);
// sf.deleteStream(streamid, ci);
}
public DecimalTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
valueStream = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA);
this.scaleStream = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.SECONDARY), true, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public static Logger doDiskRaw(StreamOperation op,
int num_operations, int val_size,
StreamFactory.StreamPhysicalType ptype,
string exp_dir)
{
Logger logger = new Logger();
if (op == StreamOperation.RandomKeyRandomValueAppend ||
op == StreamOperation.SameKeyRandomValueAppend ||
op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend)
{
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[num_operations];
for (int i = 0; i < num_operations; ++i)
{
keys[i] = new StrKey("" + i);
}
Byte[] singleval = new Byte[val_size];
random.NextBytes(singleval);
Byte[][] tmp = new Byte[num_operations][];
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
{
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[val_size];
random.NextBytes(tmp[i]);
}
}
StrKey key = new StrKey("ExpKey");
if (ptype == StreamFactory.StreamPhysicalType.File)
{
// ValueDataStream type
logger.Log("Start Stream Open");
FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
fout.Seek(0, SeekOrigin.End);
BinaryWriter fs_bw = new BinaryWriter(fout);
logger.Log("End Stream Open");
// TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
// Console.WriteLine("Time" + span.TotalSeconds);
logger.Log("Start Stream Append");
for (int i = 0; i < num_operations; ++i)
{
fs_bw.BaseStream.Seek(0, SeekOrigin.End);
/*
if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
{
fs_bw.Write(key.key);
}
else
{
*/
fs_bw.Write(keys[i].key);
/*
}
*/
fs_bw.Write(StreamFactory.NowUtc());
/*
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
fs_bw.Write(tmp[i]);
else
*/
fs_bw.Write(singleval);
fs_bw.Flush();
}
fout.Flush(true);
fs_bw.Close();
// span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
// Console.WriteLine("Time" + span.TotalSeconds);
logger.Log("End Stream Append");
}
else
{
string[] filenames = new string[num_operations];
for (int i = 0; i < num_operations; ++i)
{
filenames[i] = exp_dir + "/DiskRaw" + i;
}
// FileDataStream type
logger.Log("Start Stream Append");
for (int i = 0; i < num_operations; ++i)
{
FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
// fout.Seek(0, SeekOrigin.End);
BinaryWriter fs_bw = new BinaryWriter(fout);
// fs_bw.BaseStream.Seek(0, SeekOrigin.End);
/*
if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
{
fs_bw.Write(key.key);
}
else
{
*/
fs_bw.Write(keys[i].key);
// }
fs_bw.Write(StreamFactory.NowUtc());
/*
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
fs_bw.Write(tmp[i]);
else
*/
fs_bw.Write(singleval);
fout.Flush(true);
fs_bw.Close();
}
logger.Log("End Stream Append");
}
}
else
{
// Populate the keys and the values
Random random = new Random();
Byte[] val = new Byte[val_size];
if (ptype == StreamFactory.StreamPhysicalType.File)
{
long[] offsets = new long[num_operations];
for (int i = 0; i < num_operations; ++i)
{
offsets[i] = random.Next(0, num_operations - 1) * val_size;
}
// ValueDataStream type
logger.Log("Start Stream Open");
FileStream fin = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader fs_br = new BinaryReader(fin);
logger.Log("End Stream Open");
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
for (int i = 0; i < num_operations; ++i)
{
fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin);
fs_br.Read(val, 0, val_size);
}
fs_br.Close();
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
else
{
string[] filenames = new string[num_operations];
for (int i = 0; i < num_operations; ++i)
{
filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations);
}
// FileDataStream type
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
for (int i = 0; i < num_operations; ++i)
{
FileStream fin = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader fs_br = new BinaryReader(fin);
fs_br.Read(val, 0, val_size);
fs_br.Close();
}
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
}
CostsHelper costhelper = new CostsHelper();
logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:true)/1000.0f);
logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:false)/1000.0f);
return logger;
}
public CharTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
itemLength = schema.getMaxLength();
padding = new byte[itemLength];
}
public StringBaseTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
stringOutput = writer.createStream(id,
OrcProto.Stream.Types.Kind.DICTIONARY_DATA);
lengthOutput = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
rowOutput = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
rowIndexValueCount.Add(0L);
buildIndex = writer.buildIndex();
directStreamOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA);
directLengthOutput = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
OrcFile.WriterOptions options = writer.getOptions();
dictionaryKeySizeThreshold = options.getDictionaryKeySizeThreshold();
strideDictionaryCheck = options.getStrideDictionaryCheck();
doneDictionaryCheck = false;
}
private static TreeWriter createTreeWriter(ObjectInspector inspector,
TypeDescription schema,
StreamFactory streamFactory,
bool nullable)
{
switch (schema.getCategory())
{
case Category.BOOLEAN:
return new BooleanTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.BYTE:
return new ByteTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.SHORT:
case Category.INT:
case Category.LONG:
return new IntegerTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.FLOAT:
return new FloatTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.DOUBLE:
return new DoubleTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.STRING:
return new StringTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.CHAR:
return new CharTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.VARCHAR:
return new VarcharTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.BINARY:
return new BinaryTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.TIMESTAMP:
return new TimestampTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.DATE:
return new DateTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.DECIMAL:
return new DecimalTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.STRUCT:
return new StructTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.MAP:
return new MapTreeWriter(streamFactory.getNextColumnId(), inspector,
schema, streamFactory, nullable);
case Category.LIST:
return new ListTreeWriter(streamFactory.getNextColumnId(), inspector,
schema, streamFactory, nullable);
case Category.UNION:
return new UnionTreeWriter(streamFactory.getNextColumnId(), inspector,
schema, streamFactory, nullable);
default:
throw new ArgumentException("Bad category: " +
schema.getCategory());
}
}
public UnionTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
IList<ObjectInspector> choices = null;
if (inspector != null)
{
UnionObjectInspector insp = (UnionObjectInspector)inspector;
choices = insp.getObjectInspectors();
}
IList<TypeDescription> children = schema.getChildren();
childrenWriters = new TreeWriter[children.Count];
for (int i = 0; i < childrenWriters.Length; ++i)
{
childrenWriters[i] = createTreeWriter(
choices != null ? choices[i] : null, children[i], writer, true);
}
tags =
new RunLengthByteWriter(writer.createStream(columnId,
OrcProto.Stream.Types.Kind.DATA));
recordPosition(rowIndexPosition);
}
public TimestampTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
this.seconds = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA), true, isDirectV2, writer);
this.nanos = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.SECONDARY), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
TimeZoneInfo timeZone;
this.base_timestamp = TimeZones.GetBaseTimestamp(writer.Timezone, out timeZone);
}
public BinaryTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.stream = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.isDirectV2 = isNewWriteFormat(writer);
this.length = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
