Button submit handler for filter_admin_format_form()'s 'editor_configure' button.
function editor_form_filter_admin_format_editor_configure($form, &$form_state) {
$editor = $form_state['editor'];
if (isset($form_state['values']['editor']['editor'])) {
if ($form_state['values']['editor']['editor'] === '') {
$form_state['editor'] = FALSE;
}
elseif (empty($editor) || $form_state['values']['editor']['editor'] !== $editor->editor) {
$editor = entity_create('editor', array(
'format' => $form['#format']->format,
'editor' => $form_state['values']['editor']['editor'],
));
$form_state['editor'] = $editor;
}
}
$form_state['rebuild'] = TRUE;
}