public bool WithFileService()
{
/*
* File service initialization
*/
FileService = new BFileServiceAZ(RequiredEnvironmentVariables["AZ_STORAGE_SERVICE_URL"],
RequiredEnvironmentVariables["AZ_STORAGE_ACCOUNT_NAME"],
RequiredEnvironmentVariables["AZ_STORAGE_ACCOUNT_ACCESS_KEY"],
RequiredEnvironmentVariables["AZ_RESOURCE_GROUP_NAME"],
RequiredEnvironmentVariables["AZ_RESOURCE_GROUP_LOCATION"],
RequiredEnvironmentVariables["AZ_CLIENT_ID"],
RequiredEnvironmentVariables["AZ_CLIENT_SECRET"],
RequiredEnvironmentVariables["AZ_SUBSCRIPTION_ID"],
RequiredEnvironmentVariables["AZ_TENANT_ID"],
(string Message) =>
{
LoggingService.WriteLogs(BLoggingServiceMessageUtility.Single(EBLoggingServiceLogType.Critical, Message), ProgramID, "Initialization");
});
if (FileService == null || !FileService.HasInitializationSucceed())
{
LoggingService.WriteLogs(BLoggingServiceMessageUtility.Single(EBLoggingServiceLogType.Critical, "File service initialization has failed."), ProgramID, "Initialization");
return(false);
}
return(true);
}
public static bool GetProcessedFile(
WebServiceBaseTimeoutable _Request,
ENodeType _FileType,
IBDatabaseServiceInterface _DatabaseService,
IBFileServiceInterface _FileService,
string _CadFileStorageBucketName,
string _ModelID,
int _RevisionIndex,
out BWebServiceResponse _SuccessResponse,
out BWebServiceResponse _FailureResponse,
Action <string> _ErrorMessageAction = null)
{
_SuccessResponse = BWebResponse.InternalError("");
if (!Controller_AtomicDBOperation.Get().GetClearanceForDBOperation(_Request.InnerProcessor, ModelDBEntry.DBSERVICE_MODELS_TABLE(), _ModelID, _ErrorMessageAction))
{
_FailureResponse = BWebResponse.InternalError("Atomic operation control has failed.");
return(false);
}
var bResult = GetProcessedFile_Internal(
_FileType,
_DatabaseService,
_FileService,
_CadFileStorageBucketName,
_ModelID,
_RevisionIndex,
out _SuccessResponse,
out _FailureResponse,
_ErrorMessageAction);
Controller_AtomicDBOperation.Get().SetClearanceForDBOperationForOthers(_Request.InnerProcessor, ModelDBEntry.DBSERVICE_MODELS_TABLE(), _ModelID, _ErrorMessageAction);
return(bResult);
}
public static void Initialize(
IBDatabaseServiceInterface _DatabaseService,
IBFileServiceInterface _FileService,
string _DeploymentBranchName,
string _DeploymentBuildNumber,
string _CadProcessServiceName,
Dictionary <string, string> _FileWorkerEnvironmentVariables,
System.Action _InitFailedAction,
Action <string> _ErrorMessageAction = null)
{
FileWorkerEnvironmentVariables = _FileWorkerEnvironmentVariables;
DeploymentBranchName = _DeploymentBranchName;
DeploymentBuildNumber = _DeploymentBuildNumber;
FileService = _FileService;
DatabaseService = _DatabaseService;
Instance = new BatchProcessingCreationService();
string CadProcessServiceName = _CadProcessServiceName;
BTaskWrapper.Run(() =>
{
while (true)
{
try
{
V1Service CadProcessService = Instance.K8sManager.GetServiceByNameAndNamespace(CadProcessServiceName, SERVICE_NAMESPACE);
if (CadProcessService != null &&
CadProcessService.Status != null &&
CadProcessService.Status.LoadBalancer != null &&
CadProcessService.Status.LoadBalancer.Ingress != null &&
CadProcessService.Status.LoadBalancer.Ingress.Any()
&& CadProcessService.Spec != null &&
CadProcessService.Spec.Ports != null &&
CadProcessService.Spec.Ports.Any() &&
!string.IsNullOrWhiteSpace(CadProcessService.Status.LoadBalancer.Ingress.First().Ip))
{
CadProcessUrl = $"http://{CadProcessService.Status.LoadBalancer.Ingress.First().Ip}:{CadProcessService.Spec.Ports.First().Port}/";
WaitInit.Set();
break;
}
else
{
Thread.Sleep(1000);
}
}
catch (Exception ex)
{
_ErrorMessageAction?.Invoke($"Failed to initialize Batch process environment: {ex.Message}\n{ex.StackTrace}");
//If we fail at this point then it means the cluster master endpoint is unavailable or there is no ingress which means batch process can't system can't initialize.
//This can happen instantly or 5 minutes after the program started depending on if an ingress is still being created and how long it takes so decide what to do in provided action
_InitFailedAction.Invoke();
}
}
});
}
public Model_GetHierarchyFile_ForRevision(IBFileServiceInterface _FileService, IBDatabaseServiceInterface _DatabaseService, string _RestfulUrlParameter_ModelsKey, string _RestfulUrlParameter_RevisionsKey, string _CadFileStorageBucketName)
{
FileService = _FileService;
DatabaseService = _DatabaseService;
CadFileStorageBucketName = _CadFileStorageBucketName;
RestfulUrlParameter_ModelsKey = _RestfulUrlParameter_ModelsKey;
RestfulUrlParameter_RevisionsKey = _RestfulUrlParameter_RevisionsKey;
}
public Model_GetUnrealGeometry(IBFileServiceInterface _FileService, IBDatabaseServiceInterface _DatabaseService, string _RestfulUrlParameter_ModelsKey, string _RestfulUrlParameter_RevisionsKey, string _RestfulUrlParameter_GeometryKey, string _CadFileStorageBucketName)
{
FileService = _FileService;
DatabaseService = _DatabaseService;
CadFileStorageBucketName = _CadFileStorageBucketName;
RestfulUrlParameter_ModelsKey = _RestfulUrlParameter_ModelsKey;
RestfulUrlParameter_RevisionsKey = _RestfulUrlParameter_RevisionsKey;
RestfulUrlParameter_GeometryKey = _RestfulUrlParameter_GeometryKey;
}
public Model_GetUpdateDeleteRaw_ForRevision(IBFileServiceInterface _FileService, IBDatabaseServiceInterface _DatabaseService, string _RestfulUrlParameter_ModelsKey, string _RestfulUrlParameter_RevisionsKey, string _CadFileStorageBucketName, string _CadProcessServiceEndpoint)
{
FileService = _FileService;
DatabaseService = _DatabaseService;
CadFileStorageBucketName = _CadFileStorageBucketName;
RestfulUrlParameter_ModelsKey = _RestfulUrlParameter_ModelsKey;
RestfulUrlParameter_RevisionsKey = _RestfulUrlParameter_RevisionsKey;
CadProcessServiceEndpoint = _CadProcessServiceEndpoint;
}
public BFileServiceTest(IBFileServiceInterface _FileService, string _BucketName, string _FileKey, string _FileLocalPath, Action <string> _PrintAction)
{
SelectedFileService = _FileService;
BucketName = _BucketName;
FileKey = _FileKey;
FileLocalPath = _FileLocalPath;
PrintAction = _PrintAction;
}
public static bool GetProcessedFileNode(
WebServiceBaseTimeoutable _Request,
ENodeType _FileType,
IBDatabaseServiceInterface _DatabaseService,
IBFileServiceInterface _FileService,
string _CadFileStorageBucketName,
string _ModelID,
int _RevisionIndex,
bool _bRootNodeRequested, ulong _NodeID,
out BWebServiceResponse _SuccessResponse,
out BWebServiceResponse _FailureResponse,
Action <string> _ErrorMessageAction = null)
{
_SuccessResponse = BWebResponse.InternalError("");
uint StartIndex = 0, Size = 0;
if (!_bRootNodeRequested)
{
Convert.UniqueIDToStartIndexAndSize(_NodeID, out StartIndex, out Size);
if (StartIndex == 0 || Size == 0)
{
_FailureResponse = BWebResponse.BadRequest("Invalid Node ID.");
return(false);
}
}
if (!Controller_AtomicDBOperation.Get().GetClearanceForDBOperation(_Request.InnerProcessor, ModelDBEntry.DBSERVICE_MODELS_TABLE(), _ModelID, _ErrorMessageAction))
{
_FailureResponse = BWebResponse.InternalError("Atomic operation control has failed.");
return(false);
}
var bResult = GetProcessedFileNode_Internal(
_FileType,
_DatabaseService,
_FileService,
_CadFileStorageBucketName,
_ModelID,
_RevisionIndex,
_bRootNodeRequested,
StartIndex,
Size,
out _SuccessResponse,
out _FailureResponse,
_ErrorMessageAction);
Controller_AtomicDBOperation.Get().SetClearanceForDBOperationForOthers(_Request.InnerProcessor, ModelDBEntry.DBSERVICE_MODELS_TABLE(), _ModelID, _ErrorMessageAction);
return(bResult);
}
public bool WithFileService()
{
/*
* File service initialization
*/
FileService = new BFileServiceAWS(RequiredEnvironmentVariables["AWS_ACCESS_KEY"], RequiredEnvironmentVariables["AWS_SECRET_KEY"], RequiredEnvironmentVariables["AWS_REGION"],
(string Message) =>
{
LoggingService.WriteLogs(BLoggingServiceMessageUtility.Single(EBLoggingServiceLogType.Critical, Message), ProgramID, "Initialization");
});
if (FileService == null || !FileService.HasInitializationSucceed())
{
LoggingService.WriteLogs(BLoggingServiceMessageUtility.Single(EBLoggingServiceLogType.Critical, "File service initialization has failed."), ProgramID, "Initialization");
return(false);
}
return(true);
}
public bool WithFileService()
{
/*
* File service initialization
*/
FileService = new BFileServiceGC(RequiredEnvironmentVariables["GOOGLE_CLOUD_PROJECT_ID"],
(string Message) =>
{
LoggingService.WriteLogs(BLoggingServiceMessageUtility.Single(EBLoggingServiceLogType.Critical, Message), ProgramID, "Initialization");
});
if (FileService == null || !FileService.HasInitializationSucceed())
{
LoggingService.WriteLogs(BLoggingServiceMessageUtility.Single(EBLoggingServiceLogType.Critical, "File service initialization has failed."), ProgramID, "Initialization");
return(false);
}
return(true);
}
public GetSignedUploadUrlRequest(IBFileServiceInterface _FileService) : base()
{
FileService = _FileService;
}
public BDeleteSecretsRequest(IBFileServiceInterface _FileService, string _SecretsStorageBucket)
{
FileService = _FileService;
SecretsStorageBucket = _SecretsStorageBucket;
}
private static bool GetProcessedFileNode_Internal(
ENodeType _FileType,
IBDatabaseServiceInterface _DatabaseService,
IBFileServiceInterface _FileService,
string _CadFileStorageBucketName,
string _ModelID,
int _RevisionIndex,
bool _bRootNodeRequested, uint _NodeStartIndex, uint _NodeSize,
out BWebServiceResponse _SuccessResponse,
out BWebServiceResponse _FailureResponse,
Action <string> _ErrorMessageAction = null)
{
_SuccessResponse = BWebResponse.InternalError("");
if (!TryGettingAllInfo(
_DatabaseService,
_ModelID,
_RevisionIndex,
out ModelDBEntry ModelObject,
out Revision RevisionObject,
out int _,
out _FailureResponse,
_ErrorMessageAction))
{
return(false);
}
if (RevisionObject.FileEntry.FileProcessStage != (int)Constants.EProcessStage.Uploaded_Processed)
{
_FailureResponse = BWebResponse.NotFound("Raw file has not been processed yet.");
return(false);
}
if (_bRootNodeRequested)
{
Convert.UniqueIDToStartIndexAndSize(RevisionObject.FileEntry.ProcessedFilesRootNodeID, out _NodeStartIndex, out _NodeSize);
if (_NodeStartIndex == 0 || _NodeSize == 0)
{
_FailureResponse = BWebResponse.InternalError("Invalid Root Node ID.");
return(false);
}
}
string RelativeFileUrl = null;
switch (_FileType)
{
case ENodeType.Hierarchy:
RelativeFileUrl = RevisionObject.FileEntry.HierarchyRAFRelativeUrl;
break;
case ENodeType.Geometry:
RelativeFileUrl = RevisionObject.FileEntry.GeometryRAFRelativeUrl;
break;
case ENodeType.Metadata:
RelativeFileUrl = RevisionObject.FileEntry.MetadataRAFRelativeUrl;
break;
}
Node RetrievedNode;
var Buffer = new byte[_NodeSize];
try
{
using (var MemStream = new MemoryStream((int)_NodeStartIndex))
{
var Destination = new BStringOrStream(MemStream, _NodeSize);
if (!_FileService.DownloadFile(_CadFileStorageBucketName, RelativeFileUrl, Destination, _ErrorMessageAction, _NodeStartIndex, _NodeSize))
{
_ErrorMessageAction?.Invoke("DownloadFile has failed in GetProcessedFileNode_Internal. ModelID: " + _ModelID + ", RevisionIndex: " + _RevisionIndex + ", NodeStartIndex: " + _NodeStartIndex + ", NodeSize: " + _NodeSize);
_FailureResponse = BWebResponse.NotFound("Given Node ID does not exist.");
return(false);
}
MemStream.Seek(0, SeekOrigin.Begin);
if (MemStream.Read(Buffer, 0, (int)_NodeSize) < (int)_NodeSize)
{
_FailureResponse = BWebResponse.InternalError("Stream read operation has failed.");
return(false);
}
Convert.BufferToNode(out RetrievedNode, _FileType, Buffer, 0);
}
}
catch (Exception e)
{
_ErrorMessageAction?.Invoke("File random access/stream operations have failed. ModelID: " + _ModelID + ", RevisionIndex: " + _RevisionIndex + ", NodeStartIndex: " + _NodeStartIndex + ", NodeSize: " + _NodeSize + ", Message: " + e.Message + ", Trace: " + e.StackTrace);
_FailureResponse = BWebResponse.NotFound("Given Node ID does not exist.");
return(false);
}
if (RetrievedNode == null)
{
_FailureResponse = BWebResponse.InternalError("File node parse operation has failed.");
return(false);
}
_SuccessResponse = BWebResponse.StatusOK("Node has been located.", new JObject()
{
["node"] = JObject.Parse(JsonConvert.SerializeObject(RetrievedNode))
});
return(true);
}
private static bool GetProcessedFile_Internal(
ENodeType _FileType,
IBDatabaseServiceInterface _DatabaseService,
IBFileServiceInterface _FileService,
string _CadFileStorageBucketName,
string _ModelID,
int _RevisionIndex,
out BWebServiceResponse _SuccessResponse,
out BWebServiceResponse _FailureResponse,
Action <string> _ErrorMessageAction = null)
{
_SuccessResponse = BWebResponse.InternalError("");
if (!TryGettingAllInfo(
_DatabaseService,
_ModelID,
_RevisionIndex,
out ModelDBEntry _,
out Revision RevisionObject,
out int _,
out _FailureResponse,
_ErrorMessageAction))
{
return(false);
}
if (RevisionObject.FileEntry.FileProcessStage != (int)Constants.EProcessStage.Uploaded_Processed)
{
_FailureResponse = BWebResponse.NotFound("Raw file has not been processed yet.");
return(false);
}
string RelativeFileUrl = null;
switch (_FileType)
{
case ENodeType.Hierarchy:
RelativeFileUrl = RevisionObject.FileEntry.HierarchyCFRelativeUrl;
break;
case ENodeType.Geometry:
RelativeFileUrl = RevisionObject.FileEntry.GeometryCFRelativeUrl;
break;
case ENodeType.Metadata:
RelativeFileUrl = RevisionObject.FileEntry.MetadataCFRelativeUrl;
break;
}
if (!_FileService.CreateSignedURLForDownload(
out string DownloadUrl,
_CadFileStorageBucketName,
RelativeFileUrl,
FileEntry.EXPIRY_MINUTES,
_ErrorMessageAction))
{
_FailureResponse = BWebResponse.InternalError("Signed url generation has failed.");
return(false);
}
_SuccessResponse = BWebResponse.StatusOK("File has been located.", new JObject()
{
[FileEntry.FILE_DOWNLOAD_URL_PROPERTY] = DownloadUrl,
[FileEntry.FILE_DOWNLOAD_UPLOAD_EXPIRY_MINUTES_PROPERTY] = FileEntry.EXPIRY_MINUTES
});
return(true);
}
private static bool GetProcessedFile_Internal(
EProcessedFileType _FileType,
IBDatabaseServiceInterface _DatabaseService,
IBFileServiceInterface _FileService,
string _CadFileStorageBucketName,
string _ModelID,
int _RevisionIndex,
out BWebServiceResponse _SuccessResponse,
out BWebServiceResponse _FailureResponse,
string _GeometryId = null,
Action <string> _ErrorMessageAction = null)
{
_SuccessResponse = BWebResponse.InternalError("");
if (!TryGettingAllInfo(
_DatabaseService,
_ModelID,
_RevisionIndex,
out ModelDBEntry _,
out Revision RevisionObject,
out int _,
out _FailureResponse,
_ErrorMessageAction))
{
return(false);
}
if (RevisionObject.FileEntry.FileProcessStage != (int)Constants.EProcessStage.Uploaded_Processed)
{
_FailureResponse = BWebResponse.NotFound("Raw file has not been processed yet.");
return(false);
}
string RelativeFileUrl = null;
switch (_FileType)
{
case EProcessedFileType.HIERARCHY_CF:
RelativeFileUrl = RevisionObject.FileEntry.HierarchyCFRelativeUrl;
break;
case EProcessedFileType.HIERARCHY_RAF:
RelativeFileUrl = RevisionObject.FileEntry.HierarchyRAFRelativeUrl;
break;
case EProcessedFileType.METADATA_CF:
RelativeFileUrl = RevisionObject.FileEntry.MetadataCFRelativeUrl;
break;
case EProcessedFileType.METADATA_RAF:
RelativeFileUrl = RevisionObject.FileEntry.MetadataRAFRelativeUrl;
break;
case EProcessedFileType.GEOMETRY_CF:
RelativeFileUrl = RevisionObject.FileEntry.GeometryCFRelativeUrl;
break;
case EProcessedFileType.GEOMETRY_RAF:
RelativeFileUrl = RevisionObject.FileEntry.GeometryRAFRelativeUrl;
break;
case EProcessedFileType.UNREAL_HGM:
RelativeFileUrl = RevisionObject.FileEntry.UnrealHGMRelativeUrl;
break;
case EProcessedFileType.UNREAL_HG:
RelativeFileUrl = RevisionObject.FileEntry.UnrealHGRelativeUrl;
break;
case EProcessedFileType.UNREAL_H:
RelativeFileUrl = RevisionObject.FileEntry.UnrealHRelativeUrl;
break;
case EProcessedFileType.UNREAL_G:
if (_GeometryId == null)
{
_ErrorMessageAction?.Invoke("GeometryId was not set when tried to retrieve UnrealGeometry file (u_g)");
_FailureResponse = BWebResponse.InternalError("GeometryId was not provided.");
}
RelativeFileUrl = $"{RevisionObject.FileEntry.UnrealGRelativeUrlBasePath}{_GeometryId}.{Constants.ProcessedFileType_Extension_Map[EProcessedFileType.UNREAL_G]}";
break;
}
if (!_FileService.CreateSignedURLForDownload(
out string DownloadUrl,
_CadFileStorageBucketName,
RelativeFileUrl,
FileEntry.EXPIRY_MINUTES,
_ErrorMessageAction))
{
_FailureResponse = BWebResponse.InternalError("Signed url generation has failed.");
return(false);
}
_SuccessResponse = BWebResponse.StatusOK("File has been located.", new JObject()
{
[FileEntry.FILE_DOWNLOAD_URL_PROPERTY] = DownloadUrl,
[FileEntry.FILE_DOWNLOAD_UPLOAD_EXPIRY_MINUTES_PROPERTY] = FileEntry.EXPIRY_MINUTES
});
return(true);
}
public BatchJobCompleteRequest(IBDatabaseServiceInterface _DatabaseService, IBFileServiceInterface _FileService, IBMemoryServiceInterface _MemoryService) : base()
{
DatabaseService = _DatabaseService;
FileService = _FileService;
MemoryService = _MemoryService;
}
public StopProcessRequest(IBDatabaseServiceInterface _DatabaseService, IBFileServiceInterface _FileService) : base()
{
DatabaseService = _DatabaseService;
FileService = _FileService;
}
public void SetFileService(IBFileServiceInterface _FileService)
{
FileService = _FileService;
}
