/// <summary>
/// Ensure that the current item is pushed onto the stack.
/// </summary>
public void EnsureStack()
{
switch (_storage.Location)
{
case ItemLocation.Stack:
// Already on the stack
return;
case ItemLocation.Parameter:
case ItemLocation.Local:
case ItemLocation.Current:
PushValue();
break;
case ItemLocation.Global:
// Call method that computes the value of this global value
_helper.LoadQueryRuntime();
_helper.Call(_storage.GlobalLocation !);
break;
default:
Debug.Fail($"Invalid location: {_storage.Location}");
break;
}
_storage = _storage.ToStack();
}
private DataConnection CreateSourceDataConnection(StorageDescriptor storageDescriptor)
{
var connectionString = storageDescriptor.Tenant.HasValue
? _connectionStringSettings.GetConnectionString(
storageDescriptor.ConnectionStringIdentity, storageDescriptor.Tenant.Value)
: _connectionStringSettings.GetConnectionString(
storageDescriptor.ConnectionStringIdentity);
if (string.IsNullOrWhiteSpace(connectionString))
{
throw new Exception($"No connection string for {storageDescriptor.ConnectionStringIdentity.GetType().Name} for tenant {storageDescriptor.Tenant} is defined");
}
using var scope = new TransactionScope(TransactionScopeOption.Suppress);
// Creating connection to source that will NOT be enlisted in transactions
var connection = CreateDataConnection(
connectionString,
storageDescriptor.MappingSchema,
storageDescriptor.CommandTimeout);
if (connection.Connection.State != ConnectionState.Open)
{
connection.Connection.Open();
}
scope.Complete();
return(connection);
}
public static TableEntity ToData(CdmEntityDefinition instance, CdmCorpusContext ctx, ResolveOptions resOpt, CopyOptions options)
{
var properties = CreateTablePropertyBags(instance, resOpt, options);
var columns = new List <DataColumn>();
foreach (var attribute in instance.Attributes)
{
columns.Add(TypeAttributePersistence.ToData(attribute as CdmTypeAttributeDefinition, ctx, resOpt, options));
}
var storageDescriptor = new StorageDescriptor
{
Source = new DataSource(),
Format = new FormatInfo(),
Columns = columns
};
TableProperties teProperties = new TableProperties
{
Properties = properties,
Partitioning = new TablePartitioning(),
StorageDescriptor = storageDescriptor
};
return(new TableEntity
{
Name = instance.EntityName,
Type = SASEntityType.TABLE,
Properties = teProperties
});
}
private void ReplaceInBulk(IReadOnlyCollection <Type> dataObjectTypes, StorageDescriptor sourceStorageDescriptor, DataConnection targetConnection, IReadOnlyCollection <ICommand> replicationCommands)
{
DataConnection sourceConnection;
// Creating connection to source that will NOT be enlisted in transactions
using (var scope = new TransactionScope(TransactionScopeOption.Suppress))
{
sourceConnection = CreateDataConnection(sourceStorageDescriptor);
if (sourceConnection.Connection.State != ConnectionState.Open)
{
sourceConnection.Connection.Open();
}
scope.Complete();
}
using (sourceConnection)
{
var actorsFactory = new ReplaceDataObjectsInBulkActorFactory(dataObjectTypes, sourceConnection, targetConnection);
var actors = actorsFactory.Create();
foreach (var actor in actors)
{
var sw = Stopwatch.StartNew();
actor.ExecuteCommands(replicationCommands);
sw.Stop();
Console.WriteLine($"[{DateTime.Now}] [{Environment.CurrentManagedThreadId}] {actor.GetType().GetFriendlyName()}: {sw.Elapsed.TotalSeconds:F3} seconds");
}
}
}
/// <summary>
/// Discard the current item if it is pushed onto the stack.
/// </summary>
public void DiscardStack()
{
if (_storage.Location == ItemLocation.Stack)
{
_helper.Emit(OpCodes.Pop);
_storage = StorageDescriptor.None();
}
}
private static bool IsConfigured(StorageDescriptor descriptor) =>
!string.IsNullOrWhiteSpace(
descriptor.Tenant.HasValue
? ConnectionStringSettings.GetConnectionString(
descriptor.ConnectionStringIdentity,
descriptor.Tenant.Value)
: ConnectionStringSettings.GetConnectionString(
descriptor.ConnectionStringIdentity));
/// <summary>
/// В databaseManagementMode исключен updatestatistics - причина, т.к. будет выполнен rebuild индексов, то
/// статистика для индексов при этом будет автоматически пересчитана с FULLSCAN, нет смысла после этого делать updatestatistics
/// с меньшим SampleRate потенциально ухудшая качество статистики
/// </summary>
private static ReplicateInBulkCommand ReplicateFromDbToDbCommand(
IReadOnlyCollection <Type> typesToReplicate, StorageDescriptor from, StorageDescriptor to) =>
new ReplicateInBulkCommand(
typesToReplicate,
from,
to,
executionMode: ParallelReplication,
databaseManagementMode: DbManagementMode.DropAndRecreateConstraints |
DbManagementMode.EnableIndexManagment);
private DataConnection CreateDataConnection(StorageDescriptor storageDescriptor)
{
var connectionString = _connectionStringSettings.GetConnectionString(storageDescriptor.ConnectionStringIdentity);
var connection = SqlServerTools.CreateDataConnection(connectionString);
connection.AddMappingSchema(storageDescriptor.MappingSchema);
connection.CommandTimeout = (int)storageDescriptor.CommandTimeout.TotalMilliseconds;
return(connection);
}
/// <summary>
/// Ensure that current item is saved to the specified local variable.
/// </summary>
public void EnsureLocal(LocalBuilder bldr)
{
if (_storage.LocalLocation != bldr)
{
// Push value onto stack and then save to bldr
EnsureStack();
_helper.Emit(OpCodes.Stloc, bldr);
_storage = _storage.ToLocal(bldr);
}
}
/// <summary>
/// Set this iterator to be the same as the specified iterator.
/// </summary>
public void SetIterator(IteratorDescriptor iterInfo)
{
if (iterInfo.HasLabelNext)
{
_lblNext = iterInfo.GetLabelNext();
_hasNext = true;
}
_storage = iterInfo.Storage;
}
/// <summary>
/// Continue iteration until it is complete. Branch to "lblOnEnd" when iteration is complete.
/// </summary>
/// <remarks>
/// goto LabelNextCtxt;
/// LabelOnEnd:
/// </remarks>
public void LoopToEnd(Label lblOnEnd)
{
if (_hasNext)
{
_helper.BranchAndMark(_lblNext, lblOnEnd);
_hasNext = false;
}
// After looping is finished, storage is N/A
_storage = StorageDescriptor.None();
}
private static bool CreateStorageDescriptor(string value, out object result)
{
string dir = Path.GetDirectoryName(value);
if (string.IsNullOrEmpty(dir))
{
dir = Application.WorkerPath;
}
result = new StorageDescriptor(new PathDescriptor(dir), Path.GetFileName(value));
return(true);
}
public override void Execute()
{
StorageDescriptor pluginCache = new StorageDescriptor(Application.CacheDirectory, PluginCacheFileName, path.Name);
if (pluginCache.Load())
{
foreach (StorageContainer node in pluginCache.Nodes.Values)
{
if (!node.Validate())
{
pluginCache.Clear();
break;
}
}
}
StorageContainer assemblyNode; if (!pluginCache.TryGetNode(PluginAssemblyStorageAlias, out assemblyNode))
{
List <FileSystemDescriptor> files = Application.GetPluginFiles(path);
if (files.Count > 0)
{
FileDescriptor assemblyFile = Compile(files);
assemblyNode = pluginCache.Store(assemblyFile, PluginAssemblyStorageAlias);
foreach (FileDescriptor mixin in files)
{
pluginCache.Store(mixin);
}
pluginCache.Save();
}
else
{
Application.Warning(SeverityFlags.None, "Plugin '{0}' has no code files to compile", path.Name);
}
files = null;
}
if (assemblyNode != null && assemblyNode.Element.Exists())
{
Assembly assembly = Assembly.LoadFrom(assemblyNode.Element.GetAbsolutePath());
Type type = assembly.GetType <IPlugin>();
if (type != null)
{
instance = type.CreateInstance <IPlugin>();
if (instance != null)
{
instance.Load();
return;
}
}
Application.Warning(SeverityFlags.None, "'{0}' is not a plugin", path.GetAbsolutePath());
}
}
public ReplicateInBulkCommand(
StorageDescriptor sourceStorageDescriptor,
StorageDescriptor targetStorageDescriptor,
DbManagementMode databaseManagementMode = DbManagementMode.DropAndRecreateConstraints | DbManagementMode.EnableIndexManagment | DbManagementMode.UpdateTableStatistics,
ExecutionMode executionMode = null,
TimeSpan?bulkCopyTimeout = null)
{
SourceStorageDescriptor = sourceStorageDescriptor;
TargetStorageDescriptor = targetStorageDescriptor;
DbManagementMode = databaseManagementMode;
ExecutionMode = executionMode ?? ExecutionMode.Parallel;
BulkCopyTimeout = bulkCopyTimeout ?? DefaultBulkCopyTimeout;
}
/// <summary>
/// If the iterator has been fully cached, then iterate the values one-by-one.
/// </summary>
public void EnsureNoCache()
{
if (_storage.IsCached)
{
if (!HasLabelNext)
{
// If no Next label, this must be a singleton cache
EnsureStack();
_helper.LoadInteger(0);
_helper.CallCacheItem(_storage.ItemStorageType);
_storage = StorageDescriptor.Stack(_storage.ItemStorageType, false);
}
else
{
// int idx;
LocalBuilder locIdx = _helper.DeclareLocal("$$$idx", typeof(int));
Label lblNext;
// Make sure cache is not on the stack
EnsureNoStack("$$$cache");
// idx = -1;
_helper.LoadInteger(-1);
_helper.Emit(OpCodes.Stloc, locIdx);
// LabelNext:
lblNext = _helper.DefineLabel();
_helper.MarkLabel(lblNext);
// idx++;
_helper.Emit(OpCodes.Ldloc, locIdx);
_helper.LoadInteger(1);
_helper.Emit(OpCodes.Add);
_helper.Emit(OpCodes.Stloc, locIdx);
// if (idx >= cache.Count) goto LabelNextCtxt;
_helper.Emit(OpCodes.Ldloc, locIdx);
CacheCount();
_helper.Emit(OpCodes.Bge, GetLabelNext());
// item = cache[idx];
PushValue();
_helper.Emit(OpCodes.Ldloc, locIdx);
_helper.CallCacheItem(_storage.ItemStorageType);
SetIterator(lblNext, StorageDescriptor.Stack(_storage.ItemStorageType, false));
}
}
}
private void ExecuteInTransactionScope(StorageDescriptor storageDescriptor,
Action <DataConnection, SequentialPipelineActor> action)
{
using (var transation = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionOptions))
{
using (var targetConnection = CreateDataConnection(storageDescriptor))
{
var schemaManagenentActor =
CreateDbSchemaManagementActor((SqlConnection)targetConnection.Connection,
storageDescriptor.CommandTimeout);
action(targetConnection, schemaManagenentActor);
transation.Complete();
}
}
}
private DataConnection CreateTransactionlessDataConnection(StorageDescriptor storageDescriptor)
{
// Creating connection to source that will NOT be enlisted in transactions
using (var scope = new TransactionScope(TransactionScopeOption.Suppress))
{
var connection = CreateDataConnection(storageDescriptor);
if (connection.Connection.State != ConnectionState.Open)
{
connection.Connection.Open();
}
scope.Complete();
return(connection);
}
}
public static CdmDataPartitionDefinition FromData(CdmCorpusContext ctx, StorageDescriptor obj, string symsRootPath, FormatType formatType)
{
var newPartition = ctx.Corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef);
var symsPath = Utils.CreateSymsAbsolutePath(symsRootPath, obj.Source.Location);
newPartition.Location = Utils.SymsPathToCorpusPath(symsPath, ctx.Corpus.Storage);
var trait = Utils.CreatePartitionTrait(obj.Format.Properties, ctx, formatType);
if (trait != null)
{
newPartition.ExhibitsTraits.Add(trait);
}
else
{
Logger.Error(ctx, Tag, nameof(FromData), null, CdmLogCode.ErrPersistSymsUnsupportedTableFormat);
return(null);
}
var properties = obj.Properties;
if (properties != null)
{
if (properties.ContainsKey("cdm:name"))
{
newPartition.Name = (string)properties["cdm:name"];
}
if (properties.ContainsKey("cdm:lastFileStatusCheckTime"))
{
newPartition.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToString());
}
if (properties.ContainsKey("cdm:lastFileModifiedTime"))
{
newPartition.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToString());
}
if (properties.ContainsKey("cdm:traits"))
{
Utils.AddListToCdmCollection(newPartition.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:traits"]));
}
}
return(newPartition);
}
public static Project CreateProject(Project project, List <FileSystemDescriptor> files)
{
StorageDescriptor cache = project.Cache;
files.ParallelFor((file) =>
{
CppReferenceNode node = new CppReferenceNode(project.Target, file as FileDescriptor);
node.FetchImports();
lock (project)
project.Files.Nodes.Add(node);
StorageContainer element = cache.Store(file);
if (node.Flags.Count != 0)
{
switch ((OutputAssemblyType)node.Flags[0])
{
case OutputAssemblyType.Library: element.Tag = LibraryStorageTag; break;
case OutputAssemblyType.Console: element.Tag = ConsoleStorageTag; break;
case OutputAssemblyType.Executable: element.Tag = ExeStorageTag; break;
}
lock (project)
project.AssemblyType = (OutputAssemblyType)node.Flags[0];
}
});
ProjectAnalyzer.AddProjectFiles(project);
Filter filter = new Filter();
filter.Add(".*").Exclude = true;
project.Location.FindDirectories(filter).ParallelFor((directory) =>
{
cache.Store(directory);
});
Analyzer.SetProjectType(project);
Analyzer.SetProjectDependencies(project);
return(project);
}
private static void LoadMixins()
{
StorageDescriptor mixinCache = new StorageDescriptor(Application.CacheDirectory, ModuleCacheFileName);
if (mixinCache.Load())
{
foreach (StorageContainer node in mixinCache.Nodes.Values)
{
if (!node.Validate())
{
mixinCache.Clear();
break;
}
}
}
StorageContainer assemblyNode; if (!mixinCache.TryGetNode(MixinAssemblyStorageAlias, out assemblyNode))
{
Filter filter = new Filter();
SetMixinFilter(filter);
List <FileSystemDescriptor> mixins = Application.SdkDirectory.FindFiles(filter);
if (mixins.Count > 0)
{
FileDescriptor assemblyFile = Compile(mixins);
assemblyNode = mixinCache.Store(assemblyFile, MixinAssemblyStorageAlias);
foreach (FileDescriptor mixin in mixins)
{
mixinCache.Store(mixin);
}
mixinCache.Save();
}
filter = null;
mixins = null;
}
MixinManager.RegisterMixins(Assembly.GetExecutingAssembly());
if (assemblyNode != null && assemblyNode.Element.Exists())
{
MixinManager.RegisterMixins(Assembly.LoadFrom(assemblyNode.Element.GetAbsolutePath()));
}
}
private void UpdateAccountsCombobox()
{
m_cbAccounts.BeginUpdate();
ImageComboBoxItem item;
StorageDescriptor descriptor = null;
foreach (var account in m_configService.Accounts.OrderBy(_ => _.Type).ThenBy(_ => _.Name))
{
if (descriptor == null || (account.Type != descriptor.Type))
{
descriptor = StorageRegistry.Descriptors.SingleOrDefault(_ => _.Type == account.Type);
if (descriptor == null)
{
continue;
}
item = new ImageComboBoxItem()
{
Text = descriptor.FriendlyName,
ImageIndex = m_ilProviderIcons.Images.IndexOfKey(descriptor.Type.ToString()),
IndentLevel = 0,
Font = new Font(FontFamily.GenericSansSerif, 9, FontStyle.Bold),
};
m_cbAccounts.Items.Add(item);
}
item = new ImageComboBoxItem()
{
Text = account.Name,
ImageIndex = m_ilProviderIcons.Images.IndexOfKey(account.Type.ToString()),
Item = account,
IndentLevel = 1,
};
m_cbAccounts.Items.Add(item);
}
;
m_cbAccounts.EndUpdate();
}
public static bool FromCache(BuildTarget target, UInt32 tag, PathDescriptor location, out Project project)
{
Project tmp = new Project(tag, location);
tmp.AssemblyType = OutputAssemblyType.Static;
tmp.Target = target;
project = tmp;
if (Application.Rebuild)
{
return(false);
}
PathDescriptor cacheDirectory = new PathDescriptor(Application.CacheDirectory, tag.ToString());
if (!cacheDirectory.Exists())
{
return(false);
}
StorageDescriptor cache = tmp.Cache;
if (cache.Nodes.Count > 0)
{
Int32 rebuildFlag = 0;
cache.Nodes.Values.ParallelFor((node) =>
{
if (Interlocked.CompareExchange(ref rebuildFlag, 0, 0) == 0)
{
switch (node.Tag)
{
case 0: break;
default:
{
if (node.Tag == LibraryStorageTag)
{
lock (tmp)
tmp.AssemblyType = OutputAssemblyType.Library;
}
else if (node.Tag == ConsoleStorageTag)
{
lock (tmp)
tmp.AssemblyType = OutputAssemblyType.Console;
}
else if (node.Tag == ExeStorageTag)
{
lock (tmp)
tmp.AssemblyType = OutputAssemblyType.Executable;
}
else
{
return;
}
}
break;
}
if (node.Validate())
{
if (node.Element is FileDescriptor)
{
lock (tmp)
tmp.Files.Nodes.Add(new CppReferenceNode(target, node.Element as FileDescriptor));
}
}
else
{
Interlocked.Exchange(ref rebuildFlag, 1);
}
}
});
if (rebuildFlag == 0)
{
project.IsCached = true;
return(true);
}
}
project.Files.Clear();
project.AssemblyType = OutputAssemblyType.Library;
return(false);
}
private SqlConnection CreateSqlConnection(StorageDescriptor storageDescriptor) => new SqlConnection(_connectionStringSettings.GetConnectionString(storageDescriptor.ConnectionStringIdentity));
/// <summary>
/// Each XmlQueryType has multiple legal CLR representations. Ensure that all items returned by this iterator are in
/// the Clr representation specified by "storageTypeDest".
/// </summary>
public void EnsureItemStorageType(XmlQueryType xmlType, Type storageTypeDest)
{
// If source type = destination type, then done
if (_storage.ItemStorageType == storageTypeDest)
{
goto SetStorageType;
}
Debug.Assert(_storage.ItemStorageType == typeof(XPathItem) || storageTypeDest == typeof(XPathItem),
"EnsureItemStorageType must convert to or from Item");
// If items are cached,
if (_storage.IsCached)
{
// Check for special case of IList<XPathNavigator> -> IList<XPathItem>
if (_storage.ItemStorageType == typeof(XPathNavigator))
{
EnsureStack();
_helper.Call(XmlILMethods.NavsToItems);
goto SetStorageType;
}
// Check for special case of IList<XPathItem> -> IList<XPathNavigator>
if (storageTypeDest == typeof(XPathNavigator))
{
EnsureStack();
_helper.Call(XmlILMethods.ItemsToNavs);
goto SetStorageType;
}
}
// Iterate over each item, and convert each to the destination type
EnsureStackNoCache();
// If source type is Item,
if (_storage.ItemStorageType == typeof(XPathItem))
{
// Then downcast to Navigator
if (storageTypeDest == typeof(XPathNavigator))
{
_helper.Emit(OpCodes.Castclass, typeof(XPathNavigator));
}
else
{
// Call ValueAs methods for atomic types
_helper.CallValueAs(storageTypeDest);
}
goto SetStorageType;
}
else if (_storage.ItemStorageType == typeof(XPathNavigator))
{
// No-op if converting from XPathNavigator to XPathItem
Debug.Assert(storageTypeDest == typeof(XPathItem), "Must be converting from XPathNavigator to XPathItem");
goto SetStorageType;
}
// Destination type must be item, so generate code to create an XmlAtomicValue
_helper.LoadInteger(_helper.StaticData.DeclareXmlType(xmlType));
_helper.LoadQueryRuntime();
_helper.Call(XmlILMethods.StorageMethods[_storage.ItemStorageType].ToAtomicValue !);
SetStorageType:
_storage = _storage.ToStorageType(storageTypeDest);
}
private DataConnection CreateDataConnection(StorageDescriptor storageDescriptor)
=> CreateDataConnection(
_connectionStringSettings.GetConnectionString(storageDescriptor.ConnectionStringIdentity),
storageDescriptor.MappingSchema,
storageDescriptor.CommandTimeout);
/// <summary>
/// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
/// and applied to the table. This will end up creating a new table for each billing period.
/// </summary>
/// <param name="manifest"></param>
/// <param name="context"></param>
/// <returns>The table name</returns>
private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
{
if (String.IsNullOrEmpty(_GlueDatabaseName))
{
string Message = "No Glue database name defined, cannot create a table.";
context.LogWarning(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");
string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);
Dictionary <string, string> Parameters;
StorageDescriptor Descriptor;
switch (Format)
{
case "csv":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "skip.header.line.count", "1" },
{ "columnsOrdered", "true" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = $"{x.Category}/{x.Name}", Type = "string"
}).ToList(),
InputFormat = "org.apache.hadoop.mapred.TextInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "OpenCSVSerde",
SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
Parameters = new Dictionary <string, string>()
{
{ "escapeChar", "\\" },
{ "quoteChar", "\"" },
{ "separatorChar", "," }
}
}
};
break;
}
case "parquet":
{
Parameters = new Dictionary <string, string>()
{
{ "EXTERNAL", "TRUE" },
{ "compressionType", manifest.Compression.ToString().ToLower() },
{ "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
};
Descriptor = new StorageDescriptor()
{
Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
{
Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
}).ToList(),
InputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
Location = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
SerdeInfo = new SerDeInfo()
{
Name = "ParquetHiveSerDe",
SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
Parameters = new Dictionary <string, string>()
{
{ "serialization.format", "1" }
}
}
};
break;
}
default:
{
string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
await SNSNotify(Message, context);
return(String.Empty);
}
}
// The updated table input for this particular CUR
TableInput TblInput = new TableInput()
{
Description = Date,
Name = Date,
TableType = "EXTERNAL_TABLE",
Parameters = Parameters,
StorageDescriptor = Descriptor
};
// Make sure the database exists
GetDatabaseRequest GetDb = new GetDatabaseRequest()
{
Name = _GlueDatabaseName
};
try
{
await _GlueClient.GetDatabaseAsync(GetDb);
context.LogInfo($"Database {_GlueDatabaseName} already exists.");
}
catch (EntityNotFoundException)
{
try
{
CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
{
DatabaseInput = new DatabaseInput()
{
Name = _GlueDatabaseName
}
};
CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
}
else
{
context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
}
}
catch (Exception ex)
{
string Message = $"Failed to create the database {_GlueDatabaseName}.";
context.LogError(Message, ex);
await SNSNotify(Message + $" {ex.Message}", context);
return(String.Empty);
}
}
// Make sure the table exists
GetTableRequest GetTable = new GetTableRequest()
{
DatabaseName = _GlueDatabaseName,
Name = Date
};
try
{
GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);
UpdateTableRequest UpdateReq = new UpdateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
catch (EntityNotFoundException) // This means the table does not exist
{
CreateTableRequest CreateReq = new CreateTableRequest()
{
TableInput = TblInput,
DatabaseName = _GlueDatabaseName
};
CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);
if (Response.HttpStatusCode == HttpStatusCode.OK)
{
context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
return(TblInput.Name);
}
else
{
string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
context.LogError(Message);
await SNSNotify(Message, context);
return(String.Empty);
}
}
}
public static StorageDescriptor ToData(CdmDataPartitionDefinition instance, StorageDescriptor obj, ResolveOptions resOpt, CopyOptions options)
{
obj.Properties = new Dictionary <string, JToken>();
if (instance.Name != null)
{
obj.Properties["cdm:name"] = instance.Name;
}
if (instance.LastFileStatusCheckTime != null)
{
obj.Properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime;
}
if (instance.LastFileModifiedTime != null)
{
obj.Properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime;
}
if (instance.ExhibitsTraits != null)
{
TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
var csvTrait = tpm.FetchTraitReference("is.partition.format.CSV");
if (csvTrait != null)
{
instance.ExhibitsTraits.Remove("is.partition.format.CSV");
}
if (instance.ExhibitsTraits.Count > 0)
{
obj.Properties["cdm:traits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options),
new JsonSerializer {
NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver()
});
}
if (csvTrait != null)
{
instance.ExhibitsTraits.Add(csvTrait);
}
}
var properties = FillPropertyBagFromCsvTrait(instance);
if (properties != null)
{
obj.Format = new FormatInfo
{
InputFormat = InputFormat.OrgapachehadoopmapredSequenceFileInputFormat,
OutputFormat = OutputFormat.OrgapachehadoophiveqlioHiveSequenceFileOutputFormat,
SerializeLib = SerializeLib.Orgapachehadoophiveserde2lazyLazySimpleSerDe,
FormatType = FormatType.Csv,
Properties = properties
};
}
else
{
// error
return(null);
}
return(obj);
}
public static void DisplayManual()
{
Application.Log(SeverityFlags.None, "Forge {0}, distributed under the Schroedinger Entertainment EULA (See EULA.md for details)", Application.Version);
StorageDescriptor mixinCache = new StorageDescriptor(Application.CacheDirectory, ModuleCacheFileName);
if (mixinCache.Load() && mixinCache.Nodes.Count > 0)
{
Application.Log(SeverityFlags.None, "Loaded {0} Mixin(s) from root", mixinCache.Nodes.Count - 1);
foreach (StorageContainer mixin in mixinCache.Nodes.Values)
{
if (mixin.Tag == 0)
{
Application.Log(SeverityFlags.None, " {0}", mixin.Element.GetRelativePath(Application.SdkDirectory));
}
}
}
mixinCache = null;
Application.Log(SeverityFlags.None, "Loaded {0} Plugin(s) from root", plugins.Count);
foreach (Plugin plugin in plugins)
{
Application.Log(SeverityFlags.None, " {0}", plugin.Path.GetRelativePath(Application.SdkDirectory));
}
Application.Log(SeverityFlags.None, string.Empty);
PageFormatter page = new PageFormatter();
AutoConfig.GetAttributePage <Application>(page);
page.Sort();
PageFormatter subPage = new PageFormatter();
AutoConfig.GetAttributePage(Application.Profile.GetType(), subPage);
AutoConfig.GetAttributePage(Application.Profile, subPage);
subPage.Sort();
if (subPage.Rows.Count > 0)
{
page.AddRow("");
page.AddRow(string.Format("{0} Profile", Application.Profile.Name.ToTitleCase()));
page.Rows.AddRange(subPage.Rows);
}
Dictionary <string, PageFormatter> subSystems = new Dictionary <string, PageFormatter>();
AppDomain.CurrentDomain.GetAssemblies().ParallelFor((assembly) =>
{
foreach (Type receiver in assembly.GetTypes <IConfigReceiver>())
{
try
{
PageFormatter systemPage = new PageFormatter();
AutoConfig.GetAttributePage(receiver, systemPage);
IConfigReceiver instance; if (receiver.IsAbstract && MixinType.IsAssignableFrom(receiver))
{
instance = MixinManager.Create(receiver) as IConfigReceiver;
}
else if (!receiver.IsAbstract)
{
instance = receiver.CreateInstance <IConfigReceiver>();
}
else
{
continue;
}
if (instance != null && instance.Display)
{
instance.GetAdditionalManualInfo(systemPage);
if (systemPage.Rows.Count > 0)
{
lock (subSystems)
{
PageFormatter tmp; if (subSystems.TryGetValue(instance.DisplayName, out tmp))
{
tmp.AddRows(systemPage.Rows);
}
else
{
subSystems.Add(instance.DisplayName, systemPage);
}
}
}
}
}
catch
{ }
}
});
foreach (KeyValuePair <string, PageFormatter> systems in subSystems.OrderBy(x => x.Key))
{
systems.Value.Sort();
page.AddRow("");
page.AddRow(string.Format("{0} System", systems.Key));
page.Rows.AddRange(systems.Value.Rows);
}
foreach (Plugin plugin in plugins)
{
subPage = new PageFormatter();
plugin.GetManualPage(subPage);
subPage.Sort();
if (subPage.Rows.Count > 0)
{
page.AddRow("");
page.AddRow(string.Format("{0} Plugin", plugin.Path.Name.ToTitleCase()));
page.Rows.AddRange(subPage.Rows);
}
}
Application.Log(SeverityFlags.None, page.ToString());
}
public override void Execute()
{
LoadInput();
List <Project> projects = new List <Project>();
foreach (TaskPin inputPin in inputPins)
{
projects.Add(inputPin.Data as Project);
}
Analyzer.GetModulePaths().ParallelFor((location) =>
{
foreach (TaskPin inputPin in inputPins)
{
if ((inputPin.Data as FileDescriptor).Location.Contains(location))
{
return;
}
}
foreach (BuildTarget target in CompilerServices.Targets)
{
Project module; if (!ProjectLookup.FromCache(target, SystemTags.Cpp, location, out module))
{
List <FileSystemDescriptor> files = Application.GetProjectFiles(location, ProjectLookup.Extensions);
if (files.Count > 0)
{
module = ProjectLookup.CreateProject(module, files);
}
}
module.AssemblyType = OutputAssemblyType.Static;
module.IsModule = true;
lock (projects)
if (!projects.Contains(module))
{
Application.Log(SeverityFlags.Full, "Loaded {0} module {1}::{2}", ToolDisplayName, module.Name, module.Target.TechnicalName);
projects.Add(module);
}
}
});
projects.ParallelFor((project) =>
{
StorageDescriptor cache = project.Cache;
if (project.IsCached)
{
cache.Nodes.Values.ParallelFor((element) =>
{
if (element.Tag == ReferenceStorageTag)
{
projects.Where(x => x.Target.Id == project.Target.Id).ParallelFor((reference) =>
{
if (reference == element.Element as FileDescriptor)
{
Application.Log(SeverityFlags.Full, "Cached {0} module {1}::{2}", ToolDisplayName, reference.Name, reference.Target.TechnicalName);
lock (project)
project.References.Add(reference);
}
});
}
else if (element.Tag == ExternalStorageTag)
{
project.Externals.Add(element.Element as FileDescriptor);
}
});
}
else
{
DependencyResolver.ConnectFiles(project, projects);
foreach (Project referene in project.References)
{
cache.Store <ReferenceContainer>(referene).Tag = ReferenceStorageTag;
}
foreach (FileDescriptor referene in project.Externals)
{
cache.Store <ReferenceContainer>(referene).Tag = ExternalStorageTag;
}
cache.Save();
}
});
foreach (TaskPin inputPin in inputPins)
{
outputPins.Add(new FlaggedPin(this, inputPin.Data, SystemTags.Cpp));
}
for (int i = 0; i < inputPins.Count; i++)
{
foreach (Project reference in (outputPins[i].Data as Project).References)
{
outputPins.Add(new FlaggedPin(this, reference, SystemTags.Cpp));
}
}
if (printProjectStack)
{
PathDescriptor outputPath = Application.GetDeploymentPath(SystemTags.Analytics);
if (!outputPath.Exists())
{
try
{
outputPath.Create();
}
catch { }
}
ReferenceGraph moduleGraph = new ReferenceGraph();
foreach (Project project in projects)
{
project.Files.Nodes.ParallelFor((file) =>
{
file.Rect = Rectangle.Empty;
file.Slot = null;
});
project.Files.Save(outputPath, string.Format("{0}.cpp", project.Name));
AddProjectToGraph(moduleGraph, project);
}
moduleGraph.Save(outputPath, "CppModules");
}
}
/// <summary>
/// Set this iterator's next label and storage. This iterator will range over a set of values located in
/// "storage". To get the next value, jump to "lblNext".
/// </summary>
public void SetIterator(Label lblNext, StorageDescriptor storage)
{
_lblNext = lblNext;
_hasNext = true;
_storage = storage;
}