/*--------------------------------------------------------------------------------------------*/
private void CloseJob()
{
CommIo.Print("Saving Batch/Export results...");
long t = DateTime.UtcNow.Ticks;
StoreExportResults();
CommIo.Print("Results saved in " + GetSecs(t));
////
CommIo.Print("Saving Job results...");
t = DateTime.UtcNow.Ticks;
using (ISession sess = vSessProv.OpenSession()) {
using (ITransaction tx = sess.BeginTransaction()) {
vJob = sess.Get <Job>(vJob.Id);
vJob.TimeEnd = DateTime.UtcNow.Ticks;
sess.SaveOrUpdate(vJob);
tx.Commit();
}
}
CommIo.Print("Results saved in " + GetSecs(t));
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public ConfirmJobCommand(ICommandIo pCommIo, MatchCollection pMatches) : base(pCommIo)
{
if (pMatches.Count != 1 && pMatches.Count != 2)
{
CommIo.Print("Invalid parameter count. Expected 1 or 2 parameters.");
IsError = true;
return;
}
if (pMatches.Count == 1)
{
vJobId = -1;
return;
}
int ji;
if (!int.TryParse(pMatches[1].Value, out ji))
{
CommIo.Print("Invalid JobId parameter. Expected an integer.");
IsError = true;
return;
}
vJobId = ji;
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
private void GetItems(ISession pSess)
{
int itemCount = vBatchSize * vBatchCount;
CommIo.Print("Loading (up to) " + itemCount + " un-exported " + vTypeName + "s...");
vItemList = GetWordNetItemList(pSess, itemCount);
CommIo.Print("Found " + vItemList.Count + " " + vTypeName + "s.");
vBatchList = new List <IList <T> >();
int a = 0;
for (int i = 0; i < vBatchCount; ++i)
{
vBatchList.Add(new List <T>());
int max = Math.Min(a + vBatchSize, vItemList.Count);
for ( ; a < max; ++a)
{
vItemList[a].Note = vItemList[a].Note.Replace("&", "%26");
vBatchList[i].Add(vItemList[a]);
}
}
GC.Collect();
/*foreach ( IList<T> batch in vBatchList ) {
* CommIo.Print(" - Batch");
*
* foreach ( T item in batch ) {
* CommIo.Print(" * "+item.Name+" / "+item.ExportList.Count);
* }
* }*/
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
private void CheckBatch(Batch pBatch, ParallelLoopState pState, long pIndex)
{
string msg = "BatchId " + pBatch.Id;
if (pBatch.ExportList.Count == 0)
{
++vCheckCount;
CommIo.Print(msg + "Empty.");
return;
}
Data.Domain.Export e = pBatch.ExportList[0];
//CommIo.Print(" - First Export Id="+e.Id+", ArtifactId="+e.Artifact.Id+
// ", FabricId="+e.FabricId);
var f = new FabricClient();
f.AppDataProvSession.RefreshTokenIfNecessary();
f.UseDataProviderPerson = true;
if (!f.AppDataProvSession.IsAuthenticated)
{
throw new Exception("Could not authenticate.");
}
FabResponse <FabClass> fr =
f.Services.Traversal.GetRootStep.ClassId(e.FabricId).Get();
msg += " \t(" + (++vCheckCount) + " \tof " + vTotalCount + "): \t";
if (fr == null)
{
lock ( vFailList ) {
vFailList.Add(pBatch);
}
CommIo.Print(msg + "Failed. FabResponse was null.");
return;
}
FabClass c = fr.FirstDataItem();
if (c == null)
{
lock ( vFailList ) {
vFailList.Add(pBatch);
}
CommIo.Print(msg + "Failed. FabClass was null.");
return;
}
CommIo.Print(msg + "ArtifactId=" + c.ArtifactId + ".");
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public ConfirmAllJobsCommand(ICommandIo pCommIo, MatchCollection pMatches) : base(pCommIo)
{
if (pMatches.Count != 1)
{
CommIo.Print("Invalid parameter count. Expected 0 parameters.");
IsError = true;
return;
}
vFailList = new List <Batch>();
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public override void Start()
{
try {
CommIo.Print("Authenticating Fabric DataProvider...");
var f = new FabricClient();
f.AppDataProvSession.RefreshTokenIfNecessary();
f.UseDataProviderPerson = true;
if (!f.AppDataProvSession.IsAuthenticated)
{
throw new Exception("DataProvider is not authenticated.");
}
CommIo.Print("DataProvider authenticated.");
}
catch (Exception e) {
CommIo.Print("Authentication exception: " + e);
return;
}
var sp = new SessionProvider();
IList <Batch> batchList;
using (ISession sess = sp.OpenSession()) {
CommIo.Print("Loading all Batches...");
batchList = sess.QueryOver <Batch>()
.Fetch(x => x.ExportList).Eager
.TransformUsing(Transformers.DistinctRootEntity)
.List();
vTotalCount = batchList.Count;
vCheckCount = 0;
CommIo.Print("Found " + vTotalCount + " Batches");
}
var opt = new ParallelOptions();
opt.MaxDegreeOfParallelism = 20;
Parallel.ForEach(batchList, opt, CheckBatch);
CommIo.Print("");
CommIo.Print("Failures: " + vFailList.Count);
foreach (Batch b in vFailList)
{
CommIo.Print(" - Batch " + b.Id);
}
}
/*--------------------------------------------------------------------------------------------*/
private void RunThreads()
{
try {
CommIo.Print("Authenticating Fabric DataProvider...");
var f = new FabricClient();
f.AppDataProvSession.RefreshTokenIfNecessary();
f.UseDataProviderPerson = true;
if (!f.AppDataProvSession.IsAuthenticated)
{
throw new Exception("DataProvider is not authenticated.");
}
CommIo.Print("DataProvider authenticated.");
}
catch (Exception e) {
CommIo.Print("Authentication exception: " + e);
return;
}
vThreadStartTime = DateTime.UtcNow.Ticks;
vThreadDoneCount = 0;
vThreadSkipCount = 0;
vFailureCount = 0;
vThreadResults = new List <FabResponse <FabBatchResult> >();
////
CommIo.Print("Starting export: size=" + vBatchSize + ", count=" + vBatchCount + ", " +
"threads=" + vThreadCount + "...");
var opt = new ParallelOptions();
opt.MaxDegreeOfParallelism = vThreadCount;
Parallel.ForEach(vBatchList, opt, ThreadAction);
double time = (DateTime.UtcNow.Ticks - vThreadStartTime) / 10000000.0;
double perSec = ((vThreadDoneCount - vThreadSkipCount) * vBatchSize) / time;
CommIo.Print("Export complete: " + GetSecs(vThreadStartTime) + " fab, " +
perSec.ToString("#0.000") + " exp/sec");
////
CloseJob();
CommIo.Print("Job " + vJob.Id + " complete! Failure count: " + vFailureCount);
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public override void Start()
{
if (IsError)
{
CommIo.Print("Export cannot start due to error.");
return;
}
vSessProv = new SessionProvider();
//vSessProv.OutputSql = true;
using (ISession sess = vSessProv.OpenSession()) {
GetItems(sess);
CreateJob(sess);
RunThreads();
}
}
/*--------------------------------------------------------------------------------------------*/
protected override IList <Factor> GetWordNetItemList(ISession pSess, int pItemCount)
{
Factor facAlias = null;
CommIo.Print("Loading Factors segment...");
IList <Factor> facs = pSess.QueryOver(() => facAlias)
.JoinQueryOver <Data.Domain.Export>(x => x.ExportList, JoinType.LeftOuterJoin)
.Where(x => x.Factor == null)
.OrderBy(() => facAlias.Id).Asc
.Take(pItemCount)
.List();
CommIo.Print("Loading all Artifact Exports...");
if (ArtMap == null)
{
Artifact artAlias = null;
IList <object[]> list = pSess.QueryOver(() => artAlias)
.SelectList(sl => sl
.Select(x => x.Id)
.SelectSubQuery(
QueryOver.Of <Data.Domain.Export>()
.Where(x => x.Artifact.Id == artAlias.Id)
.Select(x => x.FabricId)
)
)
.List <object[]>();
CommIo.Print("Mapping WordNet ArtifactIds to Fabric ArtifactIds...");
ArtMap = new Dictionary <int, long>();
foreach (object[] a in list)
{
ArtMap.Add((int)a[0], (long)a[1]);
}
}
else
{
CommIo.Print(" * " + ArtMap.Keys.Count + " Cached!");
}
return(facs);
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
protected ExportBase(ICommandIo pCommIo, MatchCollection pMatches) : base(pCommIo)
{
vTypeName = typeof(T).Name;
vBatchSize = 10;
vBatchCount = 10;
vThreadCount = 5;
vDebug = false;
if (pMatches.Count != 4)
{
CommIo.Print("Invalid parameter count. Expected 3 parameters.");
IsError = true;
return;
}
int bs;
int bc;
int tc;
if (!int.TryParse(pMatches[1].Value, out bs))
{
CommIo.Print("Invalid BatchSize parameter. Expected an integer.");
IsError = true;
return;
}
if (!int.TryParse(pMatches[2].Value, out bc))
{
CommIo.Print("Invalid BatchCount parameter. Expected an integer.");
IsError = true;
return;
}
if (!int.TryParse(pMatches[3].Value, out tc))
{
CommIo.Print("Invalid ThreadCount parameter. Expected an integer.");
IsError = true;
return;
}
vBatchSize = bs;
vBatchCount = bc;
vThreadCount = tc;
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public ArtifactCommand(ICommandIo pCommIo, MatchCollection pMatches) : base(pCommIo)
{
if (pMatches.Count != 2)
{
CommIo.Print("Invalid parameter count. Expected 2 parameters.");
IsError = true;
return;
}
int ai;
if (!int.TryParse(pMatches[1].Value, out ai))
{
CommIo.Print("Invalid ArtifactId parameter. Expected an integer.");
IsError = true;
return;
}
vArtifactId = ai;
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public override void Start()
{
var sp = new SessionProvider();
using (ISession sess = sp.OpenSession()) {
Artifact a = sess.QueryOver <Artifact>()
.Where(x => x.Id == vArtifactId)
.SingleOrDefault();
Data.Domain.Export e = sess.QueryOver <Data.Domain.Export>()
.Where(x => x.Artifact.Id == vArtifactId)
.Fetch(x => x.Batch).Eager
.SingleOrDefault();
CommIo.Print("Artifact.Id: " + a.Id);
CommIo.Print("Artifact.Name: " + a.Name);
CommIo.Print("Artifact.Note: " + a.Note);
CommIo.Print("Artifact.Disamb: " + a.Disamb);
CommIo.Print("Artifact.SynsetId: " + (a.Synset == null ? "NULL" : a.Synset.Id + ""));
CommIo.Print("Artifact.WordId: " + (a.Word == null ? "NULL" : a.Word.Id + ""));
CommIo.Print("");
if (e == null)
{
CommIo.Print("Export: NULL");
}
else
{
CommIo.Print("Export.Id: " + e.Id);
CommIo.Print("Export.FabricId: " + e.FabricId);
CommIo.Print("");
CommIo.Print("Batch.Id: " + e.Batch.Id);
CommIo.Print("Batch.DbMs: " + e.Batch.DbMs);
CommIo.Print("Batch.TotalMs: " + e.Batch.TotalMs);
//CommIo.Print("Batch.Size: "+e.Batch.Size);
//CommIo.Print("Batch.Count: "+e.Batch.Count);
//CommIo.Print("Batch.Threads: "+e.Batch.Threads);
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
public override void Start()
{
try {
CommIo.Print("Authenticating Fabric DataProvider...");
var f = new FabricClient();
f.AppDataProvSession.RefreshTokenIfNecessary();
f.UseDataProviderPerson = true;
if (!f.AppDataProvSession.IsAuthenticated)
{
throw new Exception("DataProvider is not authenticated.");
}
CommIo.Print("DataProvider authenticated.");
}
catch (Exception e) {
CommIo.Print("Authentication exception: " + e);
return;
}
var sp = new SessionProvider();
using (ISession sess = sp.OpenSession()) {
if (vJobId == -1)
{
Job j = sess.QueryOver <Job>()
.OrderBy(x => x.Id).Desc
.Take(1)
.SingleOrDefault();
vJobId = j.Id;
}
CommIo.Print("Loading Job " + vJobId);
IList <Batch> batchList = sess.QueryOver <Batch>()
.Where(x => x.Job.Id == vJobId)
.List();
var failList = new List <Batch>();
int i = 0;
foreach (Batch b in batchList)
{
CommIo.Print("Confirming Batch " + b.Id + " (" + (++i) + " of " + batchList.Count + ")");
Data.Domain.Export e = b.ExportList[0];
CommIo.Print(" - First Export Id=" + e.Id + ", ArtifactId=" + e.Artifact.Id +
", FabricId=" + e.FabricId);
var f = new FabricClient();
f.AppDataProvSession.RefreshTokenIfNecessary();
f.UseDataProviderPerson = true;
if (!f.AppDataProvSession.IsAuthenticated)
{
throw new Exception("Could not authenticate.");
}
FabResponse <FabClass> fr =
f.Services.Traversal.GetRootStep.ClassId(e.FabricId).Get();
if (fr == null)
{
failList.Add(b);
CommIo.Print(" - FabResponse was null.");
continue;
}
FabClass c = fr.FirstDataItem();
if (c == null)
{
failList.Add(b);
CommIo.Print(" - FabClass was null.");
continue;
}
CommIo.Print(" - Found class: " + c.ArtifactId);
}
CommIo.Print("");
CommIo.Print("Failures: " + failList.Count);
foreach (Batch b in failList)
{
CommIo.Print(" - Batch " + b.Id);
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////
/*--------------------------------------------------------------------------------------------*/
protected void ThreadPrint(long pIndex, string pText)
{
CommIo.Print("T" + pIndex + ": \t" + pText);
}
/*--------------------------------------------------------------------------------------------*/
private FabResponse <FabBatchResult> ThreadAddItemsToFabric(IList <T> pBatch, long pIndex)
{
var f = new FabricClient();
f.AppDataProvSession.RefreshTokenIfNecessary();
f.UseDataProviderPerson = true;
if (!f.AppDataProvSession.IsAuthenticated)
{
throw new Exception("Could not authenticate.");
}
if (vDebug)
{
ThreadPrint(pIndex, "Starting batch...");
}
TBatchNew[] newBatchItems = GetNewBatchList(pBatch, pIndex);
FabResponse <FabBatchResult> fr = AddToFabric(f, newBatchItems);
if (fr == null)
{
vFailureCount += vBatchSize;
throw new Exception(" - FabResponse is null.");
}
if (fr.Error != null)
{
FabError e = fr.Error;
vFailureCount += vBatchSize;
throw new Exception(" - FabError " + e.Code + ": " + e.Name + " / " + e.Message);
}
if (fr.Data == null)
{
vFailureCount += vBatchSize;
throw new Exception(" - FabResponse.Data is null.");
}
foreach (FabBatchResult br in fr.Data)
{
if (br.Error == null)
{
continue;
}
vFailureCount++;
CommIo.Print(" # ERROR: " + br.Error.Name + " (" + br.Error.Code + "): " + br.Error.Message +
" [" + br.BatchId + " / " + br.ResultId + "]");
//Enables "repair" mode
/*if ( br.Error.Name == "UniqueConstraintViolation" ) {
* const string idStr = "ArtifactId=";
* string msg = br.Error.Message;
* int idIndex = msg.IndexOf(idStr);
*
* if ( idIndex != -1 ) {
* idIndex += idStr.Length;
* int dotIndex = msg.IndexOf(".", idIndex);
* string classId = msg.Substring(idIndex, dotIndex-idIndex);
* ThreadPrint(pIndex, "Repair: "+br.BatchId+", "+b.Id+", "+classId);
*
* var e2 = new Data.Domain.Export();
* e2.Batch = b;
* e2.FabricId = long.Parse(classId);
* SetItemTypeId(sess, e2, (int)br.BatchId);
* sess.Save(e2);
* }
* }*/
}
if (vDebug)
{
foreach (FabBatchResult br in fr.Data)
{
ThreadPrint(pIndex, " * Export success: " + vTypeName + " " +
br.BatchId + " == Fab " + br.ResultId);
}
}
return(fr);
}
