public static Stream ReadStream(string streamPath)
{
var certificate = CertificateGenerator.GetCertificateByThumbprint();
VC.Setup(null, certificate);
return(VC.ReadStream(streamPath, true));
}
public TableItem ParseFilePath(int index, string inpath)
{
string VC_path = "https://cosmos08.osdinfra.net/cosmos/bingads.algo.incubation";
TableItem res_table;
if (inpath.IndexOf(".ss") == inpath.Length - 3)
{
Stream stream = Factory.VCClientStreamFactory.OpenReadOnlyStream(VC_path + inpath);
SStreamPlusColumnGroup.IStreamMetadataReader streamMetadataReader = Factory.CreateStreamMetadataReader();
streamMetadataReader.Open(stream, VC_path + inpath);
StructuredStream.StructuredStreamSchema sstreamSchema = streamMetadataReader.Metadata.Schema;
List <string> table_name = new List <string>();
List <string> col_name = new List <string>();
List <string> col_type = new List <string>();
table_name.Add("( SSTREAM @inpath" + index.ToString() + " )");
List <ColumnInfo> col_list = sstreamSchema.ScopeSchema.Columns.ToList();
for (int i = 0; i <= col_list.Count - 1; i++)
{
col_name.Add(col_list[i].Name.ToString());
col_type.Add(col_list[i].Type.ToString().ToLower());
}
res_table = new TableItem(table_name, col_name, col_type);
}
else
{
//VcWrapper vc = new VcWrapper();
//VcWrapper VC = new VcWrapper();
Stream stream2 = VC.ReadStream(VC_path + inpath, 0, 1000, true);
using (StreamReader reader = new StreamReader(stream2))
{
int currentRow = 0;
List <string> table_name = new List <string>();
List <string> col_name = new List <string>();
List <string> col_type = new List <string>();
table_name.Add("@inpath" + index.ToString());
while (!reader.EndOfStream && currentRow < 2)
{
currentRow++;
string line = reader.ReadLine();
string[] parts = line.Split(new string[] { "\t" }, StringSplitOptions.None);
for (int i = 0; i <= parts.Length - 1; i++)
{
col_name.Add("column_" + i.ToString());
col_type.Add(DataTypeIdentifier.IdentifyToDataType(parts[i]).ToLower());
}
}
res_table = new TableItem(table_name, col_name, col_type);
}
}
return(res_table);
}
static void Main(string[] args)
{
string url = "https://cosmos09.osdinfra.net/cosmos/searchDM/local/projects/MasterData/DiscoveredDimensions/2014/06/Browser_2014_06_01.txt";
VC.SetDefaultCredentials();
using (StreamReader reader = new StreamReader(VC.ReadStream(url, false)))
{
string line, id, name;
string[] cells;
while ((line = reader.ReadLine()) != null)
{
cells = line.Split('\t');
id = cells[0]; name = cells[1];
Console.WriteLine("{0}\t{1}", id, name);
}
}
}
private static void getDates(string streamPath, ref string minDate, ref string maxDate)
{
var stream = VC.ReadStream(streamPath, false);
byte[] buffer = new byte[2048];
stream.Read(buffer, 0, 2048);
stream.Close();
string[] dates = Encoding.ASCII.GetString(buffer, 0, buffer.Length).TrimEnd('\r', '\n', '\0').Split('\t');
DateTime dtmMinDate;
DateTime.TryParse(dates[0], out dtmMinDate);
DateTime dtmMaxDate;
DateTime.TryParse(dates[1], out dtmMaxDate);
minDate = dtmMinDate.ToString("yyyyMMdd");
maxDate = dtmMaxDate.ToString("yyyyMMdd");
}
//Pass nType as parameter to determin script lookup
static void Main(string[] args)
{
FileElement fe = null;
bool bHeader = false;
LogError("Starting " + args[0]);
if (args.Length == 0)
{
LogError("No parameter passed");
System.Environment.Exit(1); // no parameters
}
else
{
try {
fe = cosmosSetup.GetFiles(args[0]);
vc = fe.VC;
script_filename = fe.script;
if (fe.excludeHeader == "1")
{
bHeader = true;
}
/*
* string baseStreamPath = fe.streamPath + DateTime.Now.ToString("yyyy-MM-dd") + '/';
*
* var stream = VC.ReadStream(fe.streamPath + fe.filePrefix + "LastUpdate.txt", false);
* byte[] buffer = new byte[2048];
* stream.Read(buffer, 0, 2048);
* stream.Close();
* string LastDate = Encoding.ASCII.GetString(buffer, 0, buffer.Length).TrimEnd('\r', '\n', '\0');
* DateTime dtmLastDate;
* if (DateTime.TryParse(LastDate, out dtmLastDate) == true)
* {
* if (dtmLastDate.Date == DateTime.Now.AddDays(-1).Date)
* {
* LogError("Script " + fe.filePrefix + " already ran for " + dtmLastDate.ToShortDateString());
* System.Environment.Exit(1); // already ran for date
* }
* startdate = dtmLastDate.AddDays(1).ToString("yyyy-MM-dd", System.Globalization.DateTimeFormatInfo.InvariantInfo);
* }
* else
* {
* startdate = DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd", System.Globalization.DateTimeFormatInfo.InvariantInfo);
* }
* //getParameters(args[0]); // pass type as parameter to program
*
* enddate = DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd", System.Globalization.DateTimeFormatInfo.InvariantInfo);
* updateScript(); //udpates start and end dates, creates temp script file and sets script_filename
*
* if (! File.Exists(script_filename))
* {
* LogError("Script file not found: " + script_filename);
* System.Environment.Exit(1); // script does not exit
* }
*/
}
catch (Exception ex)
{
LogError(ex.ToString());
System.Environment.Exit(1); // error processing parameters
}
} //end else
try
{
VcClient.VC.Setup(vc, VcClient.VC.NoProxy, null);
}
catch (Exception ex)
{
LogError("Error connecting to VC:");
LogError(ex.ToString());
System.Environment.Exit(1); // Could not connect to VC
}
try
{
/*
* var subParams = new ScopeClient.SubmitParameters(script_filename);
* ScopeClient.ScopeEnvironment.Instance.WorkingRoot = System.IO.Path.GetTempPath();
* var jobinfo = ScopeClient.Scope.Submit(subParams);
*
* // Wait
* WaitUntilJobFinished(jobinfo);
* switch (VcClient.VC.GetJobInfo(jobinfo.ID, true).State)
* {
* case JobInfo.JobState.CompletedFailure:
* LogError("Job Failed " + fe.filePrefix);
* System.Environment.Exit(1); // Job Failed
* break;
* case JobInfo.JobState.Cancelled:
* LogError("Job Cancelled " + fe.filePrefix);
* System.Environment.Exit(1); // Job Cancelled
* break;
* case JobInfo.JobState.CompletedSuccess:
* case JobInfo.JobState.Completed:
* LogError("Job Completed " + fe.filePrefix);
* break;
* default:
* LogError("Job Completed Other " + fe.filePrefix);
* System.Environment.Exit(1); // Job Completed
* break;
* }
*
* // At this point the job is finished
*/
string baseStreamPath = fe.streamPath + DateTime.Now.ToString("yyyy-MM-dd") + '/';
//var stream = VC.ReadStream(fe.streamPath + fe.filePrefix + "LastUpdate.txt", false);
var stream = VC.ReadStream(fe.streamPath + fe.updateFile, false);
byte[] buffer = new byte[2048];
stream.Read(buffer, 0, 2048);
stream.Close();
string newLastDate = Encoding.ASCII.GetString(buffer, 0, buffer.Length).TrimEnd('\r', '\n', '\0');
DateTime dtmNewLastDate;
if (fe.renameFile == "1")
{
if (DateTime.TryParse(newLastDate, out dtmNewLastDate) == true)
{
newLastDate = dtmNewLastDate.ToString("yyyyMMdd");
if (newLastDate != enddate.Replace("-", ""))
{
VC.Rename(baseStreamPath + fe.filePrefix + startdate.Replace("-", "") + '_' + enddate.Replace("-", "") + ".ss", baseStreamPath + fe.filePrefix + startdate.Replace("-", "") + '_' + newLastDate + ".ss");
}
}
}
// Add actual last update date to output filename
foreach (var streamPath in GetStreamsRecurse(baseStreamPath, new Regex(fe.filePrefix + @".*\.ss$")))
{
var uri = new Uri(streamPath);
var relativeStreamPath = uri.Segments[uri.Segments.Length - 1];
var fullCosmosPath = Path.Combine(baseStreamPath, relativeStreamPath);
DownloadCosmos.DownloadFile(baseStreamPath, relativeStreamPath.Replace(".ss", ""), fe.downloadDirectory, bHeader);
//VC.Download(fullCosmosPath, fullDiskPath, true, DownloadMode.OverWrite);
}
//DownloadCosmos.DownloadFile(fe.streamPath + DateTime.Now.ToString("yyyyMMdd") + '/', fe.filePrefix + DateTime.Now.ToString("yyyyMMdd"), "C:\\temp\\");
}
catch (VcClientExceptions.VcClientException ex)
{
LogError(ex.ToString());
}
catch (Exception ex)
{
LogError(ex.ToString());
}
File.Delete(script_filename); //delete temporary file
//Call proc for success or failure
} //end Main
//Pass nType as parameter to determin script lookup
static void Main(string[] args)
{
FileElement fe = null;
bool bHeader = false;
string baseStreamPath = string.Empty;
LogError("Starting " + args[0]);
if (args.Length == 0)
{
LogError("No parameter passed");
System.Environment.Exit(1); // no parameters
}
else
{
try {
fe = cosmosSetup.GetFiles(args[0]);
if (fe is null)
{
LogError("Parameter not found " + args[0]);
System.Environment.Exit(1); // parameter not found
}
if (args.Length == 2)
{
updateDate = args[1];
}
else
{
updateDate = DateTime.Now.ToString("yyyy-MM-dd");
}
vc = fe.VC;
script_filename = fe.script;
if (fe.excludeHeader == "1")
{
bHeader = true;
}
}
catch (Exception ex)
{
LogError(ex.ToString());
System.Environment.Exit(1); // error processing parameters
}
} //end else
try
{
VcClient.VC.Setup(vc, VcClient.VC.NoProxy, null);
}
catch (Exception ex)
{
sendEmail("Download Error Connecting to VC", fe.VC);
LogError("Error connecting to VC:");
LogError(ex.ToString());
System.Environment.Exit(1); // Could not connect to VC
}
if (fe.filePrefix.Contains("Retail"))
{
try
{
DownloadKlondike.DownloadKlondikeFile(fe.streamPath, fe.downloadDirectory, updateDate.Substring(0, 4) + "/" + updateDate.Substring(5, 2) + "/" + updateDate.Substring(8, 2) + "/");
sendEmail("Download Cosmos Complete", args[0]);
}
catch (Exception ex)
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
System.Environment.Exit(0);
}
// key is full path to stream, value is file name only
var fileDict = GetFiles.GetAllFiles(fe.streamPath, fe.filePrefix + @"\w+\.ss$");
try
{
DownloadCosmos.DownloadFileFullPath(fileDict.FirstOrDefault().Key, fileDict.FirstOrDefault().Value.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
}
catch (VcClientExceptions.VcClientException ex)
{
if (ex.ToString().Contains("throttled") || ex.ToString().Contains("ExportResetException"))
{
DownloadCosmos.DownloadFileFullPath(fileDict.FirstOrDefault().Key, fileDict.FirstOrDefault().Value.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
}
else
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("throttled") || ex.ToString().Contains("ExportResetException"))
{
DownloadCosmos.DownloadFileFullPath(fileDict.FirstOrDefault().Key, fileDict.FirstOrDefault().Value.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
}
else
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
}
System.Environment.Exit(0);
try {
baseStreamPath = fe.streamPath + updateDate + '/';
JobInfo.JobState jobStat;
if (fe.script.Length > 0)
{
jobStat = runScript.runScopeScript(fe.script);
if (jobStat != JobInfo.JobState.Completed && jobStat != JobInfo.JobState.CompletedSuccess)
{
sendEmail("Scope Failed " + args[0], runScript.errorMessage);
LogError("Scope Job Failed");
System.Environment.Exit(1);
}
}
IEnumerable <string> directories;
directories = GetDirectories(fe.streamPath);
bool pathExists = directories.Any(str => str.Contains(updateDate));
if (pathExists == false)
{
sendEmail("Download Cosmos Failed " + args[0], "No Directory " + fe.streamPath + updateDate);
LogError("Directory not found " + "No Directory " + fe.streamPath + updateDate);
System.Environment.Exit(1);
}
if (fe.renameFile == "1")
{
var stream = VC.ReadStream(fe.streamPath + fe.updateFile, false);
//var stream = VC.ReadStream(@"https://cosmos15.osdinfra.net/cosmos/dsa.email.segmentation/local/users/MeritDirect/2019-10-11/OneStore_LastUpdate_Order.txt", false);
byte[] buffer = new byte[2048];
stream.Read(buffer, 0, 2048);
stream.Close();
string[] dates = Encoding.ASCII.GetString(buffer, 0, buffer.Length).TrimEnd('\r', '\n', '\0').Split('\t');
string newLastDate = Encoding.ASCII.GetString(buffer, 0, buffer.Length).TrimEnd('\r', '\n', '\0');
// newLastDate = newLastDate.Substring(newLastDate.IndexOf('\t') + 1); removed 20191007 no longer preceding data with file name
DateTime dtmNewLastDate;
DateTime.TryParse(newLastDate, out dtmNewLastDate);
newLastDate = dtmNewLastDate.ToString("yyyyMMdd");
IEnumerable <string> streams;
streams = GetStreamsRecurse(baseStreamPath, new Regex(fe.filePrefix + @".*\.ss$"));
// Add actual last update date to output filename
if (streams.Count() > 0)
{
foreach (var streamPath in streams)
{
var uri = new Uri(streamPath);
var relativeStreamPath = uri.Segments[uri.Segments.Length - 1];
var endDate = relativeStreamPath.Substring(relativeStreamPath.LastIndexOf("_") + 1).Replace(".ss", ""); //extract date from stream name. Last token using _ (Underline) strip .ss extension
if (newLastDate != endDate && fe.renameFile == "1") //rename stream if actual end date does not match data
{
string newStreamPath = relativeStreamPath.Substring(0, relativeStreamPath.LastIndexOf("_") + 1) + dtmNewLastDate.ToString("yyyyMMdd") + ".ss";
var fullCosmosPath = Path.Combine(baseStreamPath, relativeStreamPath);
var newFullCosmosPath = Path.Combine(baseStreamPath, newStreamPath);
VC.Rename(fullCosmosPath, newFullCosmosPath);
DownloadCosmos.DownloadFile(baseStreamPath, newStreamPath.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
sendEmail("Download Cosmos Complete " + args[0], DateTime.Now.ToString("MMM dd yyyy hh:mm tt") + " " + relativeStreamPath + " " + DownloadCosmos.recordCount.ToString(" #,### records"));
}
else
{
DownloadCosmos.DownloadFile(baseStreamPath, relativeStreamPath.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
sendEmail("Download Cosmos Complete " + args[0], DateTime.Now.ToString("MMM dd yyyy hh:mm tt") + "<br/>" + relativeStreamPath + "<br/>" + DownloadCosmos.recordCount.ToString(" #,### records"));
}
}
} // streams found
else
{
sendEmail("No streams found " + fe.filePrefix, baseStreamPath);
}
} // end if rename
else
{
IEnumerable <string> streams;
streams = GetStreamsRecurse(baseStreamPath, new Regex(fe.filePrefix + @".*\.ss$"));
// Add actual last update date to output filename
if (streams.Count() > 0)
{
foreach (var streamPath in GetStreamsRecurse(baseStreamPath, new Regex(fe.filePrefix + @".*\.ss$")))
{
var uri = new Uri(streamPath);
var relativeStreamPath = uri.Segments[uri.Segments.Length - 1];
try
{
DownloadCosmos.DownloadFile(baseStreamPath, relativeStreamPath.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
}
catch (VcClientExceptions.VcClientException ex)
{
if (ex.ToString().Contains("throttled") || ex.ToString().Contains("ExportResetException"))
{
DownloadCosmos.DownloadFile(baseStreamPath, relativeStreamPath.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
}
else
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
}
catch (Exception ex)
{
if (ex.ToString().Contains("throttled") || ex.ToString().Contains("ExportResetException"))
{
DownloadCosmos.DownloadFile(baseStreamPath, relativeStreamPath.Replace(".ss", ""), fe.downloadDirectory, bHeader, fe.incr);
}
else
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
}
sendEmail("Download Cosmos Complete " + args[0], DateTime.Now.ToString("MMM dd yyyy hh:mm tt") + "<br/>" + relativeStreamPath + "<br/>" + DownloadCosmos.recordCount.ToString(" #,##0 records"));
}
}
else
{
sendEmail("No streams found " + args[0], baseStreamPath);
}
}
LogError("Completed " + args[0]);
}
catch (VcClientExceptions.VcClientException ex)
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
catch (Exception ex)
{
sendEmail("DownloadCosmos Error " + args[0], ex.ToString());
LogError(ex.ToString());
System.Environment.Exit(2);
}
System.Environment.Exit(0);
} //end Main
