//
// The ISchema interface
//
public VxSchemaErrors Put(VxSchema schema, VxSchemaChecksums sums,
VxPutOpts opts)
{
log.print("Put\n");
bool no_retry = (opts & VxPutOpts.NoRetry) != 0;
int old_err_count = -1;
IEnumerable<string> keys = schema.Keys;
VxSchemaErrors errs = new VxSchemaErrors();
// Sometimes we'll get schema elements in the wrong order, so retry
// until the number of errors stops decreasing.
while (errs.Count != old_err_count)
{
log.print("Calling Put on {0} entries\n",
old_err_count == -1 ? schema.Count : errs.Count);
old_err_count = errs.Count;
errs.Clear();
List<string> tables = new List<string>();
List<string> nontables = new List<string>();
foreach (string key in keys)
{
if (schema[key].type == "Table")
tables.Add(key);
else
nontables.Add(key);
}
errs.Add(PutSchemaTables(tables, schema, sums, opts));
foreach (string key in nontables)
{
log.print("Calling PutSchema on {0}\n", key);
VxSchemaError e = PutSchemaElement(schema[key], opts);
if (e != null)
errs.Add(key, e);
}
// If we only had one schema element, retrying it isn't going to
// fix anything. We retry to fix ordering problems.
if (no_retry || errs.Count == 0 || schema.Count == 1)
break;
log.print("Got {0} errors, old_errs={1}, retrying\n",
errs.Count, old_err_count);
keys = errs.Keys.ToList();
}
return errs;
}
VxSchemaError VxPutSchema(string type, string name, string text,
VxPutOpts opts)
{
VxSchemaElement elem = VxSchemaElement.create(type, name, text, false);
VxSchema schema = new VxSchema(elem);
VxSchemaErrors errs = VxPutSchema(schema, opts);
if (errs == null || errs.Count == 0)
return null;
WVPASSEQ(errs.Count, 1);
// Just return the first error
foreach (var p in errs)
return p.Value[0];
WVFAIL("Shouldn't happen: couldn't find error to return");
return null;
}
//
// The ISchema interface
//
// Note: this implementation ignores the sums.
public VxSchemaErrors Put(VxSchema schema, VxSchemaChecksums sums,
VxPutOpts opts)
{
WvDbusMsg call = methodcall("PutSchema",
String.Format("{0}i", VxSchema.GetDbusSignature()));
WvDbusWriter writer = new WvDbusWriter();
schema.WriteSchema(writer);
writer.Write((int)opts);
call.Body = writer.ToArray();
WvDbusMsg reply = bus.send_and_wait(call);
if (reply.signature == VxSchemaErrors.GetDbusSignature())
return new VxSchemaErrors(reply.iter().pop());
else
reply.check(VxSchemaErrors.GetDbusSignature());
return null;
}
public void TestTableUpdate(string tabname, string tabschema,
VxPutOpts opts, string expected_warning)
{
VxSchema schema = new VxSchema();
schema.Add("Table", tabname, tabschema, false);
VxSchemaErrors errs = dbus.Put(schema, null, opts);
log.print("Received errors: {0}\n", errs.ToString());
int errcount = 0;
int warncount = 0;
foreach (var errlist in errs)
foreach (var err in errlist.Value)
{
log.print("Got error, err.level={0}\n", err.level);
if (err.level == WvLog.L.Error)
errcount++;
else if (err.level == WvLog.L.Warning)
warncount++;
}
WVPASSEQ(errcount, 0);
WVPASSEQ(warncount, expected_warning.ne() ? 1 : 0);
WVPASSEQ(errs.Count, errcount + warncount);
if (expected_warning.ne())
{
string key = "Table/" + tabname;
WVPASSEQ(errs[key][0].key, key);
WVPASSEQ(errs[key][0].msg, expected_warning);
WVPASSEQ(errs[key][0].errnum, -1);
WVPASSEQ((int)errs[key][0].level, (int)WvLog.L.Warning);
WVPASSEQ(errs[key].Count, 1);
}
schema = dbus.Get("Table/" + tabname);
WVPASSEQ(schema.Count, 1);
CheckTable(schema, tabname, tabschema);
}
//
// The ISchemaBackend interface
//
// Export the current schema to the backing directory, in a format that can
// be read back later.
public VxSchemaErrors Put(VxSchema schema, VxSchemaChecksums sums,
VxPutOpts opts)
{
bool isbackup = (opts & VxPutOpts.IsBackup) != 0;
DirectoryInfo dir = new DirectoryInfo(exportdir);
dir.Create();
foreach (var p in schema)
{
if (!sums.ContainsKey(p.Key))
throw new ArgumentException("Missing checksum for " + p.Key);
VxSchemaElement elem = p.Value;
if (elem.text == null || elem.text == "")
DropSchema(new string[] {elem.key});
else
ExportToDisk(p.Value, sums[p.Key], isbackup);
}
// Writing schemas to disk doesn't give us any per-element errors.
return new VxSchemaErrors();
}
// Replaces the named object in the database. elem.text is a verbatim
// hunk of text returned earlier by GetSchema. 'destructive' says whether
// or not to perform potentially destructive operations while making the
// change, e.g. dropping a table so we can re-add it with the right
// columns.
private VxSchemaError PutSchemaElement(VxSchemaElement elem, VxPutOpts opts)
{
try
{
bool destructive = (opts & VxPutOpts.Destructive) != 0;
if (destructive || elem.type != "Table")
{
try {
DropSchema(elem.key);
} catch (VxSqlException e) {
// Check if it's a "didn't exist" error, rethrow if not.
// SQL Error 3701 means "can't drop sensible item because
// it doesn't exist or you don't have permission."
// SQL Error 15151 means "can't drop XML Schema collection
// because it doesn't exist or you don't have permission."
if (!e.ContainsSqlError(3701) && !e.ContainsSqlError(15151))
throw;
}
}
if (elem.text.ne())
{
log.print("Putting element: {0}...\n",
elem.ToSql().shorten(60));
DbiExec(elem.ToSql());
}
}
catch (VxRequestException e)
{
log.print("Got error from {0}: {1}\n", elem.key, e.Message);
return new VxSchemaError(elem.key, e);
}
return null;
}
private VxSchemaErrors PutSchemaTable(VxSchemaTable curtable,
VxSchemaTable newtable, VxPutOpts opts)
{
bool destructive = (opts & VxPutOpts.Destructive) != 0;
string tabname = newtable.name;
string key = newtable.key;
var diff = VxSchemaTable.GetDiff(curtable, newtable);
var coladd = new List<VxSchemaTableElement>();
var coldel = new List<VxSchemaTableElement>();
var colchanged = new List<VxSchemaTableElement>();
var otheradd = new List<VxSchemaTableElement>();
var otherdel = new List<VxSchemaTableElement>();
foreach (var kvp in diff)
{
VxSchemaTableElement elem = kvp.Key;
VxDiffType difftype = kvp.Value;
if (elem.elemtype == "primary-key" || elem.elemtype == "index")
{
if (difftype == VxDiffType.Add)
otheradd.Add(elem);
else if (difftype == VxDiffType.Remove)
otherdel.Add(elem);
else if (difftype == VxDiffType.Change)
{
// We don't want to bother trying to change indexes or
// primary keys; it's easier to just delete and re-add
// them.
otherdel.Add(curtable[elem.GetElemKey()]);
otheradd.Add(elem);
}
}
else
{
if (difftype == VxDiffType.Add)
coladd.Add(elem);
else if (difftype == VxDiffType.Remove)
coldel.Add(elem);
else if (difftype == VxDiffType.Change)
colchanged.Add(elem);
}
}
var errs = new VxSchemaErrors();
// Might as well check this sooner rather than later.
if (!destructive && coldel.Count > 0)
{
List<string> colstrs = new List<string>();
foreach (var elem in coldel)
colstrs.Add(elem.GetParam("name"));
// Sorting this is mostly unnecessary, except it makes life a lot
// nicer in the unit tests.
colstrs.Sort();
string errmsg = wv.fmt("Refusing to drop columns ([{0}]) " +
"when the destructive option is not set.",
colstrs.join("], ["));
errs.Add(key, new VxSchemaError(key, errmsg, -1));
goto done;
}
// Perform any needed column changes.
// Note: we call dbi.execute directly, instead of DbiExec, as we're
// running SQL we generated ourselves so we shouldn't blame any
// errors on the client's SQL. We'll catch the DbExceptions and
// turn them into VxSchemaErrors.
var deleted_indexes = new List<VxSchemaTableElement>();
var added_columns = new List<VxSchemaTableElement>();
bool transaction_started = false;
bool transaction_resolved = false;
try
{
// Delete any to-delete indexes first, to get them out of the way.
// Indexes are easy to deal with, they don't cause data loss.
// Note: we can't do this inside the transaction, MSSQL doesn't
// let you change columns that used to be covered by the dropped
// indexes. Instead we'll drop the indexes outside the
// transaction, and restore them by hand if there's an error.
foreach (var elem in otherdel)
{
log.print("Dropping {0}\n", elem.ToString());
string idxname = elem.GetParam("name");
// Use the default primary key name if none was specified.
if (elem.elemtype == "primary-key" && idxname.e())
idxname = curtable.GetDefaultPKName();
var err = DropSchemaElement("Index/" + tabname + "/" + idxname);
if (err != null)
{
errs.Add(key, err);
goto done;
}
deleted_indexes.Add(elem);
}
// If an ALTER TABLE query fails inside a transaction, the
// transaction is automatically rolled back, even if you start
// an inner transaction first. This makes error handling
// annoying. So before we start the real transaction, try to make
// the column changes in a test transaction that we'll always roll
// back to see if they'd fail.
var ErrWhenAltering = new Dictionary<string, VxSchemaError>();
foreach (var elem in colchanged)
{
VxSchemaError err = null;
log.print("Doing a trial run of modifying {0}\n",
elem.GetElemKey());
dbi.execute("BEGIN TRANSACTION coltest");
try
{
// Try to change the column the easy way, without dropping
// or adding anything and without any expected errors.
var change_errs = ApplyChangedColumn(newtable,
curtable[elem.GetElemKey()], elem, null, VxPutOpts.None);
if (change_errs.Count > 0)
err = change_errs[newtable.key][0];
}
catch (SqlException e)
{
// OK, the easy way doesn't work. Remember the error for
// when we do it for real.
log.print("Caught exception in trial run: {0} ({1})\n",
e.Message, e.Number);
err = new VxSchemaError(key, e);
}
log.print("Rolling back, err='{0}'\n",
err == null ? "" : err.ToString());
DbiExecRollback("coltest");
ErrWhenAltering.Add(elem.GetElemKey(), err);
}
log.print("About to begin real transaction\n");
// Add new columns before deleting old ones; MSSQL won't let a
// table have no data columns in it, even temporarily.
// Do this outside the transaction since failures here will
// automatically cause a rollback, even if we handle them.
// It's easy enough for us to roll back by hand if needed.
foreach (var elem in coladd)
{
log.print("Adding {0}\n", elem.ToString());
string add_format = "ALTER TABLE [{0}] ADD {1}\n";
string query = wv.fmt(add_format,
tabname, newtable.ColumnToSql(elem, true));
try
{
dbi.execute(query);
}
catch (SqlException e)
{
// Error 4901: adding a column on a non-empty table failed
// due to neither having a default nor being nullable.
// Don't try anything special in destructive mode, just
// fail and nuke the table.
if (!destructive && e.Number == 4901)
{
log.print("Couldn't add a new non-nullable column " +
"without a default. Making column nullable.\n");
var nullable = GetNullableColumn(elem);
string nullquery = wv.fmt(add_format,
tabname, newtable.ColumnToSql(nullable, true));
log.print("Executing {0}", nullquery);
dbi.execute(nullquery);
}
else
throw;
}
added_columns.Add(elem);
}
transaction_started = true;
dbi.execute("BEGIN TRANSACTION TableUpdate");
foreach (var elem in coldel)
{
log.print("Dropping {0}\n", elem.ToString());
DropTableColumn(newtable, elem);
}
foreach (var elem in colchanged)
{
var expected_err = ErrWhenAltering[elem.GetElemKey()];
var change_errs = ApplyChangedColumn(newtable,
curtable[elem.GetElemKey()], elem, expected_err, opts);
if (change_errs != null && change_errs.Count > 0)
{
errs.Add(change_errs);
goto done;
}
}
// Now that all the columns are finalized, add in any new indices.
foreach (var elem in otheradd)
{
log.print("Adding {0}\n", elem.ToString());
VxSchemaError err = PutSchemaTableIndex(key, curtable, elem);
if (err != null)
{
errs.Add(key, err);
goto done;
}
}
log.print("All changes made, committing transaction.\n");
dbi.execute("COMMIT TRANSACTION TableUpdate");
transaction_resolved = true;
}
catch (SqlException e)
{
var err = new VxSchemaError(key, e);
log.print("Caught exception: {0}\n", err.ToString());
errs.Add(key, err);
}
finally
{
if (transaction_started && !transaction_resolved)
{
log.print("Transaction failed, rolling back.\n");
if (transaction_started)
DbiExecRollback("TableUpdate");
foreach (var elem in added_columns)
{
log.print("Restoring {0}\n", elem.ToString());
try
{
DropTableColumn(newtable, elem);
}
catch (SqlException e)
{
log.print("Caught error clearing column: {0}\n",
e.Message);
}
}
foreach (var elem in deleted_indexes)
{
log.print("Restoring index {0}\n", elem.ToString());
var err = PutSchemaTableIndex(key, curtable, elem);
if (err != null)
errs.Add(key, err);
}
}
}
// Check for null entries in columns that are supposed to be non-null
if (errs.Count == 0)
{
foreach (var elem in newtable)
{
string nullity = elem.GetParam("null");
if (elem.elemtype == "column" && nullity.ne() && nullity != "1")
{
string colname = elem.GetParam("name");
string query = wv.fmt("SELECT count(*) FROM [{0}] " +
"WHERE [{1}] IS NULL",
tabname, colname);
int num_nulls = -1;
try
{
num_nulls = dbi.select_one(query);
}
catch (SqlException e)
{
string errmsg = wv.fmt(
"Couldn't figure out if '{0}' has nulls: {1}",
colname, e.Message);
log.print(errmsg + "\n");
errs.Add(key, new VxSchemaError(
key, errmsg, -1, WvLog.L.Warning));
}
if (num_nulls > 0)
{
string errmsg = wv.fmt("Column '{0}' was requested " +
"to be non-null but has {1} null elements.",
colname, num_nulls);
log.print(errmsg + "\n");
errs.Add(key, new VxSchemaError(
key, errmsg, -1, WvLog.L.Warning));
}
}
}
}
done:
return errs;
}
private VxSchemaErrors ApplyChangedColumn(VxSchemaTable table,
VxSchemaTableElement oldelem, VxSchemaTableElement newelem,
VxSchemaError expected_err, VxPutOpts opts)
{
VxSchemaErrors errs = new VxSchemaErrors();
log.print("Altering {0}\n", newelem.ToString());
bool destructive = (opts & VxPutOpts.Destructive) != 0;
string colname = newelem.GetParam("name");
// Remove any old default constraint; even if it doesn't change, it
// can get in the way of modifying the column. We'll add it again
// later if needed.
if (oldelem.HasDefault())
{
string defquery = wv.fmt("ALTER TABLE [{0}] DROP CONSTRAINT {1}",
table.name, table.GetDefaultDefaultName(colname));
log.print("Executing {0}\n", defquery);
dbi.execute(defquery);
}
bool did_default_constraint = false;
// Don't try to alter the table if we know it won't work.
if (expected_err == null)
{
string query = wv.fmt("ALTER TABLE [{0}] ALTER COLUMN {1}",
table.name, table.ColumnToSql(newelem, false));
log.print("Executing {0}\n", query);
dbi.execute(query);
}
else
{
// Some table attributes can't be changed by ALTER TABLE,
// such as changing identity values, or data type changes that
// would truncate data. If the client has set the Destructive
// flag though, we can try to drop and re-add the column.
if (destructive)
{
log.print("Alter column would fail, dropping and adding.\n");
log.print("Expected error message: {0} ({1})\n",
expected_err.msg, expected_err.errnum);
string delquery = wv.fmt("ALTER TABLE [{0}] " +
"DROP COLUMN [{1}]",
table.name, colname);
// We need to include the default value here (the second
// parameter to ColumnToSql), otherwise adding a column to a
// table with data in it might not work.
string addquery = wv.fmt("ALTER TABLE [{0}] ADD {1}",
table.name, table.ColumnToSql(newelem, true));
log.print("Executing {0}\n", delquery);
dbi.execute(delquery);
log.print("Executing {0}\n", addquery);
dbi.execute(addquery);
did_default_constraint = true;
}
else
{
// Error 515: Can't modify a column because it contains nulls
// and the column requires non-nulls.
if (expected_err.errnum == 515)
{
log.print("Couldn't modify column due to null " +
"restriction. Making column nullable.\n");
var nullable = GetNullableColumn(newelem);
string query = wv.fmt("ALTER TABLE [{0}] ALTER COLUMN {1}",
table.name, table.ColumnToSql(nullable, false));
log.print("Executing {0}\n", query);
dbi.execute(query);
}
else
{
log.print("Can't alter table and destructive flag " +
"not set. Giving up.\n");
string key = table.key;
string errmsg = wv.fmt("Refusing to drop and re-add " +
"column [{0}] when the destructive option " +
"is not set. Error when altering was: '{1}'",
colname, expected_err.msg);
errs.Add(key, new VxSchemaError(key, errmsg, -1));
}
}
}
// No errors so far, let's try to add the new default values if we
// didn't do it already.
// FIXME: Check for actual errors, don't care about warnings.
if (errs.Count == 0 && newelem.HasDefault() && !did_default_constraint)
{
string defquery = wv.fmt("ALTER TABLE [{0}] ADD CONSTRAINT {1} " +
"DEFAULT {2} FOR {3}",
table.name, table.GetDefaultDefaultName(colname),
newelem.GetParam("default"), colname);
log.print("Executing {0}\n", defquery);
dbi.execute(defquery);
}
if (errs.Count != 0)
log.print("Altering column had errors: " + errs.ToString());
return errs;
}
private VxSchemaErrors PutSchemaTables(List<string> tables,
VxSchema newschema, VxSchemaChecksums newsums, VxPutOpts opts)
{
VxSchema curschema = Get(tables);
VxSchemaErrors errs = new VxSchemaErrors();
foreach (string key in tables)
{
log.print("Putting table {0}\n", key);
string curtype = curschema.ContainsKey(key) ?
curschema[key].type : "Table";
string newtype = newschema.ContainsKey(key) ?
newschema[key].type : "Table";
if (newtype != "Table" || curtype != "Table")
throw new ArgumentException("PutSchemaTables called on " +
"non-table element '" + key + "'.");
// Check for the easy cases, an all-new table or table deletion
if (!curschema.ContainsKey(key))
{
// New table, let PutSchemaElement handle it like before.
VxSchemaError e = PutSchemaElement(newschema[key], opts);
if (e != null)
errs.Add(key, e);
continue;
}
if (!newschema.ContainsKey(key))
{
// Deleted table, let DropSchemaElement deal with it.
VxSchemaError e = DropSchemaElement(key);
if (e != null)
errs.Add(key, e);
continue;
}
// An existing table has been modified.
VxSchemaTable newtable;
VxSchemaTable curtable;
if (newschema[key] is VxSchemaTable)
newtable = (VxSchemaTable)newschema[key];
else
newtable = new VxSchemaTable(newschema[key]);
if (curschema[key] is VxSchemaTable)
curtable = (VxSchemaTable)curschema[key];
else
curtable = new VxSchemaTable(curschema[key]);
VxSchemaErrors put_table_errs = null;
put_table_errs = PutSchemaTable(curtable, newtable, opts);
// If anything goes wrong updating a table in destructive mode,
// drop and re-add it. We want to be sure the schema is updated
// exactly.
bool destructive = (opts & VxPutOpts.Destructive) != 0;
if (destructive && put_table_errs.Count > 0)
{
put_table_errs = null;
log.print("Couldn't cleanly modify table '{0}'. Dropping " +
"and re-adding it.\n", newtable.name);
VxSchemaError e = PutSchemaElement(newschema[key], opts);
if (e != null)
errs.Add(key, e);
}
if (put_table_errs != null && put_table_errs.Count > 0)
errs.Add(put_table_errs);
}
return errs;
}
VxSchemaErrors VxPutSchema(VxSchema schema, VxPutOpts opts)
{
log.print(" + VxPutSchema\n");
return dbus.Put(schema, null, opts);
}
public void TestTableUpdateError(string tabname, string tabschema,
string errmsg, string oldval, VxPutOpts opts)
{
TestTableUpdateError(tabname, tabschema, errmsg, oldval, opts, -1,
WvLog.L.Error);
}
public void TestTableUpdate(string tabname, string tabschema,
VxPutOpts opts)
{
TestTableUpdate(tabname, tabschema, opts, null);
}
public void TestTableUpdateError(string tabname, string tabschema,
string errmsg, string oldval, VxPutOpts opts, int errno, WvLog.L level)
{
string key = "Table/" + tabname;
VxSchema schema = new VxSchema();
schema.Add("Table", tabname, tabschema, false);
VxSchemaErrors errs = dbus.Put(schema, null, opts);
log.print("Received errors: {0}\n", errs.ToString());
WVPASSEQ(errs.Count, 1);
WVPASSEQ(errs[key][0].key, key);
log.print("Expected error prefix: [{0}]\n", errmsg);
WVPASS(errs[key][0].msg.StartsWith(errmsg));
WVPASSEQ(errs[key][0].errnum, errno);
WVPASSEQ((int)errs[key][0].level, (int)level);
WVPASSEQ(errs[key].Count, 1);
// Ensure that we didn't break what was already there.
schema = dbus.Get(key);
if (oldval.ne())
{
WVPASSEQ(schema.Count, 1);
CheckTable(schema, tabname, oldval);
}
else
WVPASSEQ(schema.Count, 0);
}
