private static void Main()
{
byte[] data = null;
var r53Client = new AmazonS3Client(AwsAccessKeyId, AwsSecretAccessKey, EndPoint);
ListObjectsRequest request = new ListObjectsRequest();
request.BucketName = BucketName;
ListObjectsResponse response = r53Client.ListObjects(request);
if (response.IsTruncated)
{
// Set the marker property
request.Marker = response.NextMarker;
}
else
{
request = null;
}
foreach (S3Object obj in response.S3Objects)
{
S3Bucket bk = new S3Bucket();
S3FileInfo s3f = new S3FileInfo(r53Client, BucketName, obj.Key);
TryGetFileData(obj.Key, ref data);
string result = System.Text.Encoding.UTF8.GetString(data);
Console.WriteLine("data - " + result);
//File.Create("D:\\Mail.text").Write(data,0,0);
Log_text_File(result);
var ddf = s3f.OpenText();
var teest = (ddf.BaseStream);
var ss = s3f.OpenRead();
using (StreamReader reader = new StreamReader(ddf.BaseStream))
{
Console.WriteLine(reader.ReadLine());
}
Console.WriteLine("Object - " + obj.Key);
Console.WriteLine(" Size - " + obj.Size);
Console.WriteLine(" LastModified - " + obj.LastModified);
Console.WriteLine(" Storage class - " + obj.StorageClass);
}
Console.ReadKey();
}
public string[] ReadAllLines(string bucketName, string objectKey)
{
List <String> lines = new List <string>();
S3FileInfo file = new S3FileInfo(client, bucketName, objectKey);
if (file.Exists)
{
StreamReader reader = file.OpenText();
string line = null;
while ((line = reader.ReadLine()) != null)
{
lines.Add(line);
}
}
return(lines.ToArray());
}
public static List <double> getBoostingNormThresholdList(string filename)
{
if (!Form1.UseS3 && !File.Exists(filename))
{
MessageBox.Show(@"the file " + Path.GetFileName(filename) + @" doesnt exist in " + Path.GetFullPath(filename));
return(null);
}
StreamReader sr;
if (Form1.UseS3)
{
string dir_name = Path.GetDirectoryName(filename);
string file_name = Path.GetFileName(filename);
S3DirectoryInfo s3dir = new S3DirectoryInfo(Form1.S3client, Form1.bucketName, dir_name);
S3FileInfo artFile = s3dir.GetFile(file_name);
sr = artFile.OpenText();
}
else
{
sr = new StreamReader(File.OpenRead(filename));
}
string line = sr.ReadLine();
List <double> NormArry = new List <double>();
if (line != null)
{
string[] values = line.Split(Form1.seperator, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < values.Count(); i++)
{
NormArry.Add(double.Parse(values[i]));
}
sr.Close();
}
return(NormArry);
}
public double[][] getDataTableTMP(string filename)
{
StreamReader reader;
long lineCount = 0;
if (Form1.UseS3)
{
string dir_name = Path.GetDirectoryName(filename);
string file_name = Path.GetFileName(filename);
S3DirectoryInfo s3dir = new S3DirectoryInfo(Form1.S3client, Form1.bucketName, dir_name);
S3FileInfo artFile = s3dir.GetFile(file_name);
if (artFile.Exists == false && file_name.Contains("Valid"))
{
string tmp = file_name.Replace("Valid", "testing");
artFile = s3dir.GetFile(tmp);
}
bool faileEread = true;
reader = null;
while (faileEread)
{
try
{
reader = artFile.OpenText();
while (!reader.EndOfStream)
{
reader.ReadLine();
lineCount++;
}
reader.DiscardBufferedData();
reader.BaseStream.Seek(0, SeekOrigin.Begin);
reader.BaseStream.Position = 0;
faileEread = false;
}
catch
{
faileEread = true;
}
}
//reader = artFile.OpenText();
//reader = artFile.OpenRead();
//while (!reader.EndOfStream)
//{
// reader.ReadLine();
// lineCount++;
//}
//reader.DiscardBufferedData();
//reader.BaseStream.Seek(0, SeekOrigin.Begin);
//reader.BaseStream.Position = 0;
}
else
{
if (!File.Exists(filename))//IF NO VALID EXISTS - TRY WITH TEST
{
filename = filename.Replace("Valid", "testing");
}
reader = new StreamReader(File.OpenRead(filename));
lineCount = File.ReadAllLines(filename).Length;
}
//GET THE FIRST LINE
string line = reader.ReadLine();
string[] values = line.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
//IF NO VALUES ALERT
if (values.Count() < 1)
{
return(null);
}
double[][] dt = new double[lineCount][];
dt[0] = new double[values.Count()];
for (int j = 0; j < values.Count(); j++)
{
dt[0][j] = double.Parse(values[j]);
}
//SET VALUES TO TABLE
int counter = 1;
while (!reader.EndOfStream)
{
line = reader.ReadLine();
values = line.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
dt[counter] = new double[values.Count()];
for (int j = 0; j < values.Count(); j++)
{
double tmp;
if (double.TryParse(values[j], out tmp))
{
dt[counter][j] = tmp;
}
else
{
dt[counter][j] = -1;
}
}
counter++;
}
reader.Close();
return(dt);
}
public double[][] getDataTableWithNan(string filename, ref Dictionary <Tuple <int, int>, bool> naTable)
{
StreamReader reader;
long lineCount = 0;
List <double> artVal = new List <double>();
List <int> emptyVal = new List <int>();
string tmpline = "";
#region first loop
if (Form1.UseS3)
{
string dir_name = Path.GetDirectoryName(filename);
string file_name = Path.GetFileName(filename);
S3DirectoryInfo s3dir = new S3DirectoryInfo(Form1.S3client, Form1.bucketName, dir_name);
S3FileInfo artFile = s3dir.GetFile(file_name);
if (artFile.Exists == false && file_name.Contains("Valid"))
{
string tmp = file_name.Replace("Valid", "testing");
artFile = s3dir.GetFile(tmp);
}
bool faileEread = true;
reader = null;
while (faileEread)
{
try
{
reader = artFile.OpenText();
while (!reader.EndOfStream)
{
tmpline = reader.ReadLine();
lineCount++;
//***********************
//haqndle missing values
if (lineCount == 1)//first read
{
string[] tmpvalues = tmpline.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < tmpvalues.Count(); i++)
{
double tmpDouble;
if (double.TryParse(tmpvalues[i], out tmpDouble))
{
artVal.Add(tmpDouble);
}
else
{
artVal.Add(0);
emptyVal.Add(i);
}
}
}
if (emptyVal.Count > 0) // not all artVal values are set - at least once...
{
string[] tmpvalues = tmpline.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < emptyVal.Count(); i++)
{
double tmpDouble;
if (double.TryParse(tmpvalues[emptyVal[i]], out tmpDouble))
{
artVal[emptyVal[i]] = tmpDouble;
emptyVal.RemoveAt(i);
}
}
}
//***********************
}
reader.DiscardBufferedData();
reader.BaseStream.Seek(0, SeekOrigin.Begin);
reader.BaseStream.Position = 0;
faileEread = false;
}
catch
{
faileEread = true;
}
}
}
else
{
if (!File.Exists(filename))//IF NO VALID EXISTS - TRY WITH TEST
{
filename = filename.Replace("Valid", "testing");
}
reader = new StreamReader(File.OpenRead(filename));
lineCount = File.ReadAllLines(filename).Length;
StreamReader tmpreader = new StreamReader(File.OpenRead(filename));
bool stoploop = false;
bool firstLine = true;
while (!tmpreader.EndOfStream && !stoploop)
{
tmpline = tmpreader.ReadLine();
if (firstLine)//first read
{
firstLine = false;
string[] tmpvalues = tmpline.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < tmpvalues.Count(); i++)
{
double tmpDouble;
if (double.TryParse(tmpvalues[i], out tmpDouble))
{
artVal.Add(tmpDouble);
}
else
{
artVal.Add(0);
emptyVal.Add(i);
}
}
}
if (emptyVal.Count > 0) // not all artVal values are set - at least once...
{
string[] tmpvalues = tmpline.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < emptyVal.Count(); i++)
{
double tmpDouble;
if (double.TryParse(tmpvalues[emptyVal[i]], out tmpDouble))
{
artVal[emptyVal[i]] = tmpDouble;
emptyVal.RemoveAt(i);
}
}
}
//***********************
if (emptyVal.Count < 1)
{
stoploop = true;
}
}
tmpreader.Close();
}
#endregion
if (emptyVal.Count > 0)
{
MessageBox.Show("there is a column with only Nan values - stop and repair");
}
//GET THE FIRST LINE
string line = reader.ReadLine();
string[] values = line.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
//IF NO VALUES ALERT
if (values.Count() < 1)
{
return(null);
}
double[][] dt = new double[lineCount][];
dt[0] = new double[values.Count()];
for (int j = 0; j < values.Count(); j++)
{
double tmpDouble = 0;
if (double.TryParse(values[j], out tmpDouble))
{
dt[0][j] = double.Parse(values[j]);
}
else
{
dt[0][j] = artVal[j];
naTable.Add(new Tuple <int, int>(0, j), true);
}
}
//SET VALUES TO TABLE
int counter = 1;
while (!reader.EndOfStream)
{
line = reader.ReadLine();
values = line.Split(seperator, StringSplitOptions.RemoveEmptyEntries);
dt[counter] = new double[values.Count()];
for (int j = 0; j < values.Count(); j++)
{
double tmpDouble = 0;
if (double.TryParse(values[j], out tmpDouble))
{
dt[counter][j] = double.Parse(values[j]);
}
else
{
dt[counter][j] = artVal[j];
naTable.Add(new Tuple <int, int>(counter, j), true);
}
}
counter++;
}
reader.Close();
return(dt);
}
public static List <GeoWave> getConstWaveletsFromFile(string filename, recordConfig rc)
{
if (!Form1.UseS3 && !File.Exists(filename))//this func was not debugged after modification
{
MessageBox.Show(@"the file " + Path.GetFileName(filename) + @" doesnt exist in " + Path.GetFullPath(filename));
return(null);
}
StreamReader sr;
if (Form1.UseS3)
{
string dir_name = Path.GetDirectoryName(filename);
string file_name = Path.GetFileName(filename);
S3DirectoryInfo s3dir = new S3DirectoryInfo(Form1.S3client, Form1.bucketName, dir_name);
S3FileInfo artFile = s3dir.GetFile(file_name);
sr = artFile.OpenText();
}
else
{
sr = new StreamReader(File.OpenRead(filename));
}
string[] values = { "" };
string line;
string DimensionReductionMatrix = "";
int numOfWavlets = -1;
int dimension = -1;
int labelDimension = -1;
double approxOrder = -1;
while (!sr.EndOfStream && values[0] != "StartReading")
{
line = sr.ReadLine();
values = line.Split(Form1.seperator, StringSplitOptions.RemoveEmptyEntries);
if (values[0] == "DimensionReductionMatrix")
{
DimensionReductionMatrix = values[1];
}
else if (values[0] == "numOfWavlets")
{
numOfWavlets = int.Parse(values[1]);
}
else if (values[0] == "approxOrder")
{
approxOrder = int.Parse(values[1]);
}
else if (values[0] == "dimension")
{
dimension = int.Parse(values[1]);
}
else if (values[0] == "labelDimension")
{
labelDimension = int.Parse(values[1]);
}
else if (values[0] == "StartReading")
{
;
}
else
{
MessageBox.Show(@"the file " + Path.GetFileName(filename) + @" already exist in " + Path.GetFullPath(filename) + @" might have bad input !");
}
}
//read values
List <GeoWave> gwArr = new List <GeoWave>();
while (!sr.EndOfStream)
{
GeoWave gw = new GeoWave(dimension, labelDimension, rc);
line = sr.ReadLine();
if (line != null)
{
values = line.Split(Form1.seperator, StringSplitOptions.RemoveEmptyEntries);
}
gw.ID = int.Parse(values[0]);
gw.child0 = int.Parse(values[1]);
gw.child1 = int.Parse(values[2]);
int counter = 0;
for (int j = 0; j < dimension; j++)
{
gw.boubdingBox[0][j] = int.Parse(values[3 + 4 * j]);//the next are the actual values and not the indeces int the maingrid - so we skip 4 elementsat a time
gw.boubdingBox[1][j] = int.Parse(values[4 + 4 * j]);
counter = 4 + 2 * 4;
}
gw.level = int.Parse(values[counter + 1]);
counter = counter + 2;
for (int j = 0; j < labelDimension; j++)
{
gw.MeanValue[j] = double.Parse(values[counter + j]);
counter++;
}
gw.norm = double.Parse(values[counter]);
gw.parentID = int.Parse(values[counter + 1]);
gwArr.Add(gw);
}
sr.Close();
return(gwArr);
}
