This won’t work because of the way jQuery handles the button name (can be with or without quotes)
This will work:
var button_name="Test";
var dialog_buttons = {};
dialog_buttons[button_name] = function(){ closeInstanceForm(Function); }
dialog_buttons['Cancel'] = function(){ $(this).dialog('close'); }
$('#instanceDialog').dialog({ buttons: dialog_buttons });