void WriteCreateTable(TextWriter writer, TableDataSchema table, string name = null, bool asType = false)
{
name = name ?? Escape(table.Name);
var i = 0;
if (asType)
{
writer.Write("CREATE TYPE {0} AS TABLE (", name);
}
else
{
writer.Write("CREATE TABLE {0} (", name);
}
foreach (var field in table.Fields)
{
writer.WriteLine(i++ > 0 ? "," : "");
writer.Write(" [{0}] {1} {2}", field.Name,
GetColumnType(field.ValueType, field.FieldType == FieldType.Key ? 60 : (int?)null),
field.FieldType == FieldType.Key || (field.ValueType.IsValueType && Nullable.GetUnderlyingType(field.ValueType) == null) ? "NOT NULL" : "NULL");
}
if (table.Keys.Length > 0)
{
writer.WriteLine(i++ > 0 ? "," : "");
//IGNORE_DUP_KEY to ignore hash collisions.
writer.Write(" PRIMARY KEY CLUSTERED ({0}) WITH (IGNORE_DUP_KEY = ON)", string.Join(",", table.Keys.Select(pos => string.Format("[{0}]", pos.Value.Name))));
}
writer.WriteLine();
writer.WriteLine(")");
writer.WriteLine();
}
public override ITableDataWriter CreateTableDataWriter(TableDataSchema table)
{
var csvTable = _owner.CreateTableData(table, Path.Combine(Directory, table.Name + ".txt"));
AddTableData(csvTable);
return(csvTable);
}
public override ITableDataWriter CreateTableDataWriter(TableDataSchema schema)
{
var data = new BinaryTableData(schema, Path.Combine(Directory, schema.Name + ".bin"));
AddTableData(data);
return(data);
}
public static string GetJoinToFactAncestor(TableDataSchema schema, string ancestorCriteria)
{
var sb = new StringBuilder();
GetJoinToFactAncestor(schema, ancestorCriteria, sb);
return(sb.ToString());
}
public static void GetJoinToFactAncestor(TableDataSchema schema, string ancestorCriteria, StringBuilder sb)
{
var parent = schema.RelatedTables.FirstOrDefault(r => r.RelationType == RelationType.Parent);
sb.AppendLine();
if (parent == null)
{
sb.Append("WHERE ").Append(ancestorCriteria);
}
else
{
sb.AppendFormat("INNER JOIN [{0}] ON ", parent.RelatedTable.Name);
for (var i = 0; i < parent.Fields.Count; i++)
{
if (i > 0)
{
sb.Append(" AND ");
}
sb.AppendFormat("[{0}].[{1}] = [{2}].[{3}]",
parent.RelatedTable.Name, parent.RelatedFields[i].Name,
schema.Name, parent.Fields[i].Name);
}
GetJoinToFactAncestor(parent.RelatedTable, ancestorCriteria, sb);
}
}
public CsvTableData(TableDataSchema schema, string path, string delimiter = "\t", CultureInfo numberCulture = null)
: base(schema)
{
Path = path;
Delimiter = delimiter;
NumberCulture = numberCulture ?? CultureInfo.CurrentCulture;
DateFormat = "yyyy-MM-dd";
DateTimeFormat = "yyyy-MM-dd HH:mm:ss";
}
private WritableTableData CreateTableData(TableDataSchema schema, string path)
{
if (BinaryOutput)
{
return(new BinaryTableData(schema, path.Replace(".txt", ".bin")));
}
return(new CsvTableData(schema, path,
Delimiter, _numberCulture));
}
public RowComparer(TableDataSchema schema)
{
foreach (var field in GetSortFields(schema))
{
_comparers.Add(new ComparerWrapper(field.Value.ValueType.GetComparer(),
field.Value.SortOrder == SortOrder.Descending)
.AsIndexed <IComparer>(field.Index));
}
_keys = schema.Fields.FindIndices(schema.IsKey).Select(ix => ix.Index).ToArray();
}
static string GetFieldName(TableDataSchema table, string fieldName)
{
var field = table.Fields.FirstOrDefault(f => f.Name == fieldName);
if (field == null)
{
throw new KeyNotFoundException(string.Format("No field with name '{0}' in table '{1}", fieldName, table.Name));
}
return(string.IsNullOrEmpty(field.FriendlyName) ? field.Name : field.FriendlyName);
}
public static Tuple <TableDataSchema, PartitionField> GetPartitionField(TableDataSchema schema)
{
if (schema.IsCenterTable())
{
var field = schema.Fields.OfType <PartitionField>().FirstOrDefault();
return(field != null ? new Tuple <TableDataSchema, PartitionField>(schema, field) : null);
}
var parent = schema.RelatedTables.FirstOrDefault(r => r.RelationType == RelationType.Parent);
return(parent != null?GetPartitionField(parent.RelatedTable) : null);
}
public virtual Field GetKeyField(TableDataSchema schema)
{
var name = FormatName(schema, null);
var postfix = 1;
while (schema.Fields.Any(f => f.Name == name))
{
name = FormatName(schema, postfix++);
}
return(new Field {
Name = name, FieldType = FieldType.Key, ValueType = typeof(TKey)
});
}
public static IEnumerable <Indexed <Field> > GetSortFields(TableDataSchema schema)
{
var fields = schema.Fields;
//Order by fields where SortOrder is specified, then by keys (always order by something)
foreach (var ix in fields.AsIndexed().OrderBy(ix => ix.Value.SortOrder == SortOrder.Unspecified).ThenBy(ix => ix.Index))
{
var field = ix.Value;
if (field.SortOrder != SortOrder.Unspecified || schema.IsKey(field))
{
yield return(ix);
}
}
}
public static string GetUpdateCriteria(TableDataSchema table, Tuple <TableDataSchema, PartitionField> partitionField, bool stale, DateTime?date = null, DateTime?cutoff = null)
{
var field = string.Format("[{0}].[{1}]", partitionField.Item1.Name, partitionField.Item2.Name);
var refDate = GetPartitionDate(date ?? DateTime.UtcNow.Add(-partitionField.Item2.StaleTime), partitionField.Item2.StaleTime);
var criteria = new StringBuilder(field);
criteria.Append(stale ? ">=" : "<");
criteria.AppendFormat("'{0}'", refDate.ToString("o"));
if (cutoff.HasValue && !stale)
{
criteria.AppendFormat(" AND {0} >= '{1}'", field, cutoff.Value.ToString("o"));
}
return(GetJoinToFactAncestor(table, criteria.ToString()));
}
protected virtual string FormatName(TableDataSchema schema, int?postfix)
{
return(schema.Name + "Id" + (postfix.HasValue ? "" + postfix : ""));
}
protected TableData(TableDataSchema schema)
{
Schema = schema;
}
static string GetFieldFriendlyName(TableDataSchema table, string fieldName)
{
return(GetFieldName(table, fieldName));
}
string GetTransientPartitionName(TableDataSchema schema)
{
return(schema.Name + "_Transient");
}
public MergedTableData(TableDataSchema schema, IEnumerable <TableData> sources = null)
: base(schema)
{
Sources = sources != null?sources.ToList() : new List <TableData>();
}
static string PostFixMeasureName(TableDataSchema table, string name)
{
var hasParent = table.RelatedTables.Any(r => r.RelationType == RelationType.Parent);
return(hasParent ? name + " (" + table.Name + ")" : name);
}
public BinaryTableData(TableDataSchema schema, string path)
: base(schema)
{
Path = path;
}
protected WritableTableData(TableDataSchema schema) : base(schema)
{
}
/// <summary> /// Adds a table with the specified schema to this partition and returns a <see cref="ITableDataWriter"/> for writing data to it /// </summary> /// <param name="schema">The schema to add a table for</param> /// <returns></returns> public abstract ITableDataWriter CreateTableDataWriter(TableDataSchema schema);
