From 52398a6b569c8de55b4e4ee931c82e47e2ee4b31 Mon Sep 17 00:00:00 2001 From: jim-p Date: Fri, 30 May 2014 14:10:18 -0400 Subject: [PATCH] Bring in proper gmirror support for the GUI and notifications. Made a general gmirror library to perform various gmirror tasks and get information, using some of the former widget logic to start. Updated widget to use this new code. Added a Diag > GEOM Mirrors page that displays information about existing mirrors and perform various management tasks. Current actions include rebuilding a drive, forgetting disconnected mirror drives, insert/remove, deactivate/activate, clearing medatada. It's now possible to use the GUI to rebuild a failed mirror by performing a forget, then insert action to replace a missing/dead drive. Also included is a notification setup. Mirror status is polled every 60 seconds, and if any aspect of the mirror changes, notifications are issues that alert in the GUI and by SMTP, etc. --- etc/inc/gmirror.inc | 313 ++++++++++++++++ etc/rc | 9 + usr/local/sbin/gmirror_status_check.php | 92 +++++ usr/local/www/diag_gmirror.php | 344 ++++++++++++++++++ usr/local/www/fbegin.inc | 4 + .../www/widgets/include/gmirror_status.inc | 85 ----- .../widgets/widgets/gmirror_status.widget.php | 2 +- 7 files changed, 763 insertions(+), 86 deletions(-) create mode 100644 etc/inc/gmirror.inc create mode 100644 usr/local/sbin/gmirror_status_check.php create mode 100644 usr/local/www/diag_gmirror.php delete mode 100644 usr/local/www/widgets/include/gmirror_status.inc diff --git a/etc/inc/gmirror.inc b/etc/inc/gmirror.inc new file mode 100644 index 0000000000..917427d389 --- /dev/null +++ b/etc/inc/gmirror.inc @@ -0,0 +1,313 @@ + 0) { + /* Loop through gmirror status output. */ + foreach ($status as $line) { + /* Split the line by whitespace */ + $all = preg_split("/[\s\t]+/", trim($line), 3); + if (count($all) == 3) { + /* If there are three items on a line, it is mirror name, status, and component */ + $currentmirror = basename($all[0]); + $mirrors[$currentmirror]['name'] = basename($all[0]); + $mirrors[$currentmirror]['status'] = $all[1]; + if (!is_array($mirrors[$currentmirror]['components'])) + $mirrors[$currentmirror]['components'] = array(); + $mirrors[$currentmirror]['components'][] = $all[2]; + } + } + } + /* Return an hash of mirrors and components */ + return $mirrors; +} + +/* Get only status word for a single mirror. */ +function gmirror_get_status_single($mirror) { + $status = ""; + $mirror_status = gmirror_get_status(); + var_dump($mirror_status); + return $mirror_status[$mirror]['status']; +} + +/* Generate an HTML formatted status for mirrors and disks in a small format for the widget */ +function gmirror_html_status() { + $mirrors = gmirror_get_status(); + $output = ""; + if (count($mirrors) > 0) { + $output .= "\n"; + $output .= "Name\n"; + $output .= "Status\n"; + $output .= "Component\n"; + $output .= "\n"; + foreach ($mirrors as $mirror => $name) { + $components = count($name["components"]); + $output .= "\n"; + $output .= "{$name['name']}\n"; + $output .= "{$name['status']}\n"; + $output .= "{$name['components'][0]}\n"; + $output .= "\n"; + if (count($name["components"]) > 1) { + $morecomponents = array_slice($name["components"], 1); + foreach ($morecomponents as $component) { + $output .= "\n"; + $output .= "{$component}\n"; + $output .= "\n"; + } + } + } + } else { + $output .= "No Mirrors Found\n"; + } + // $output .= "Updated at " . date("F j, Y, g:i:s a") . "\n"; + return $output; +} + +/* List all disks in the system (potential gmirror targets) */ +function gmirror_get_disks() { + $disklist = ""; + /* Get a list of disks in a scriptable way, exclude optical drives */ + exec("/sbin/geom disk status -s | /usr/bin/grep -v '[[:blank:]]*cd[[:digit:]]*' | /usr/bin/awk '{print $1;}'", $disklist); + return $disklist; +} + +/* List all potential gmirror consumers */ +function gmirror_get_unused_consumers() { + $consumerlist = ""; + /* Get a list of consumers, exclude existing mirrors and diskid entries */ + exec("/sbin/geom part status -s | /usr/bin/egrep -v '(mirror|diskid)' | /usr/bin/awk '{print $1, $3;}'", $consumerlist); + $all_consumers = array(); + foreach ($consumerlist as $cl) { + $parts = explode(" ", $cl); + foreach ($parts as $part) + $all_consumers[] = $part; + } + return $all_consumers; +} + +/* List all existing geom mirrors */ +function gmirror_get_mirrors() { + $mirrorlist = ""; + exec("/sbin/gmirror list | /usr/bin/grep '^Geom name:' | /usr/bin/awk '{print $3;}'", $mirrorlist); + return $mirrorlist; +} + + +/* List all consumers for a given mirror */ +function gmirror_get_consumers_in_mirror($mirror) { + if (!is_valid_mirror($mirror)) + return array(); + + $consumers = array(); + exec("/sbin/gmirror status -s " . escapeshellarg($mirror) . " | /usr/bin/awk '{print $3;}'", $consumers); + return $consumers; +} + +/* Test if a given consumer is a member of an existing mirror */ +function is_consumer_in_mirror($consumer, $mirror) { + if (!is_valid_consumer($consumer) || !is_valid_mirror($mirror)) + return false; + + $mirrorconsumers = gmirror_get_consumers_in_mirror($mirror); + return in_array(basename($consumer), $mirrorconsumers); +} + +/* Test if a mirror exists */ +function is_valid_mirror($mirror) { + $mirrors = gmirror_get_mirrors(); + return in_array($mirror, $mirrors); +} + +/* Test if a disk is valid/exists */ +function is_valid_disk($disk) { + $adisks = gmirror_get_disks(); + return in_array(basename($disk), $adisks); +} + +/* Test if a consumer is valid and in use in a mirror */ +function is_consumer_used($consumer) { + $found = false; + $mirrors = gmirror_get_mirrors(); + foreach ($mirrors as $mirror) { + $consumers = gmirror_get_consumers_in_mirror($mirror); + if (in_array($consumer, $consumers)) + return true; + } + return false; +} + +/* Test if a consumer is valid and not in use */ +function is_consumer_unused($consumer) { + $consumers = gmirror_get_unused_consumers(); + return in_array($consumer, $consumers); +} + +/* Test if a consumer is valid (either a disk or partition) */ +function is_valid_consumer($consumer) { + return (is_consumer_unused($consumer) || is_consumer_used($consumer)); +} + +/* Remove all disconnected drives from a mirror */ +function gmirror_forget_disconnected($mirror) { + if (!is_valid_mirror($mirror)) + return false; + return mwexec("/sbin/gmirror forget " . escapeshellarg($mirror)); +} + +/* Insert another consumer into a mirror */ +function gmirror_insert_consumer($mirror, $consumer) { + if (!is_valid_mirror($mirror) || !is_valid_consumer($consumer)) + return false; + return mwexec("/sbin/gmirror insert " . escapeshellarg($mirror) . " " . escapeshellarg($consumer)); +} + +/* Remove consumer from a mirror and clear its metadata */ +function gmirror_remove_consumer($mirror, $consumer) { + if (!is_valid_mirror($mirror) || !is_valid_consumer($consumer)) + return false; + return mwexec("/sbin/gmirror remove " . escapeshellarg($mirror) . " " . escapeshellarg($consumer)); +} + +/* Wipe geom info from drive (if mirror is not running) */ +function gmirror_clear_consumer($consumer) { + if (!is_valid_consumer($consumer)) + return false; + return mwexec("/sbin/gmirror clear " . escapeshellarg($consumer)); +} + +/* Find the balance method used by a given mirror */ +function gmirror_get_mirror_balance($mirror) { + if (!is_valid_mirror($mirror)) + return false; + $balancemethod = ""; + exec("/sbin/gmirror list " . escapeshellarg($mirror) . " | /usr/bin/grep '^Balance:' | /usr/bin/awk '{print $2;}'", $balancemethod); + return $balancemethod[0]; +} + +/* Change balance algorithm of the mirror */ +function gmirror_configure_balance($mirror, $balancemethod) { + if (!is_valid_mirror($mirror) || !in_array($balancemethod, $balancemethods)) + return false; + return mwexec("/sbin/gmirror configure -b " . escapeshellarg($balancemethod) . " " . escapeshellarg($mirror)); +} + +/* Force a mirror member to rebuild */ +function gmirror_force_rebuild($mirror, $consumer) { + if (!is_valid_mirror($mirror) || !is_valid_consumer($consumer)) + return false; + return mwexec("/sbin/gmirror rebuild " . escapeshellarg($mirror) . " " . escapeshellarg($consumer)); +} + +/* Show all metadata on the physical consumer */ +function gmirror_get_consumer_metadata($consumer) { + if (!is_valid_consumer($consumer)) + return array(); + $output = ""; + exec("/sbin/gmirror dump " . escapeshellarg($consumer), $output); + return array_map('trim', $output); +} + +/* Test if a consumer has metadata, indicating it is a member of a mirror (active or inactive) */ +function gmirror_consumer_has_metadata($consumer) { + return (count(gmirror_get_consumer_metadata($consumer)) > 0); +} + +/* Find the mirror to which this consumer belongs */ +function gmirror_get_consumer_metadata_mirror($consumer) { + if (!is_valid_consumer($consumer)) + return array(); + $metadata = gmirror_get_consumer_metadata($consumer); + foreach ($metadata as $line) { + if (substr($line, 0, 5) == "name:") { + list ($key, $value) = explode(":", $line, 2); + return trim($value); + } + } +} + +/* Deactivate consumer, removing it from service in the mirror, but leave metadata intact */ +function gmirror_deactivate_consumer($mirror, $consumer) { + if (!is_valid_mirror($mirror) || !is_valid_consumer($consumer)) + return false; + return mwexec("/sbin/gmirror deactivate " . escapeshellarg($mirror) . " " . escapeshellarg($consumer)); +} + +/* Reactivate a deactivated consumer */ +function gmirror_activate_consumer($mirror, $consumer) { + if (!is_valid_mirror($mirror) || !is_valid_consumer($consumer)) + return false; + return mwexec("/sbin/gmirror activate " . escapeshellarg($mirror) . " " . escapeshellarg($consumer)); +} + +/* Find the size of the given mirror */ +function gmirror_get_mirror_size($mirror) { + if (!is_valid_mirror($mirror)) + return false; + $mirrorsize = ""; + exec("/sbin/gmirror list " . escapeshellarg($mirror) . " | /usr/bin/grep 'Mediasize:' | /usr/bin/head -n 1 | /usr/bin/awk '{print $2;}'", $mirrorsize); + return $mirrorsize[0]; +} + +/* Return a list of all potential consumers on a disk with sizes. The geom part + list output is a little odd, we can't get the output for just the disk, if the disk contains + slices those get output also. */ +function gmirror_get_all_unused_consumer_sizes_on_disk($disk) { + if (!is_valid_disk($disk) || !is_consumer_unused($disk)) + return array(); + $output = ""; + exec("/sbin/geom part list " . escapeshellarg($disk) . " | /usr/bin/egrep '(Name:|Mediasize:)' | /usr/bin/cut -c4- | /usr/bin/sed -l -e 'N;s/\\nMediasize://;P;D;' | /usr/bin/cut -c7-", $output); + $disk_contents = array(); + foreach ($output as $line) { + list($name, $size, $humansize) = explode(" ", $line, 3); + $consumer = array(); + $consumer['name'] = $name; + $consumer['size'] = $size; + $consumer['humansize'] = $humansize; + $disk_contents[] = $consumer; + } + return $disk_contents; +} + +/* Get only the size for one specific potential consumer. */ +function gmirror_get_unused_consumer_size($consumer) { + $consumersizes = gmirror_get_all_unused_consumer_sizes_on_disk($consumer); + foreach ($consumersizes as $csize) { + if ($csize['name'] == $consumer) + return $csize['size']; + } + return -1; +} +?> \ No newline at end of file diff --git a/etc/rc b/etc/rc index 01df7d20ff..0675ea3dea 100755 --- a/etc/rc +++ b/etc/rc @@ -443,6 +443,15 @@ echo "done." /bin/chmod a+rw /tmp/. +# Check for GEOM mirrors +GMIRROR_STATUS=`/sbin/gmirror status` +if [ "${GMIRROR_STATUS}" != "" ]; then + # Using a flag file at bootup saves an expensive exec/check on each page load. + /usr/bin/touch /var/run/gmirror_active + # Setup monitoring/notifications + /usr/local/bin/minicron 60 /var/run/gmirror_status_check.pid /usr/local/sbin/gmirror_status_check.php +fi + echo "Bootup complete" /usr/local/bin/beep.sh start 2>&1 >/dev/null diff --git a/usr/local/sbin/gmirror_status_check.php b/usr/local/sbin/gmirror_status_check.php new file mode 100644 index 0000000000..d507963f16 --- /dev/null +++ b/usr/local/sbin/gmirror_status_check.php @@ -0,0 +1,92 @@ +#!/usr/local/bin/php -f + 0) { + // Check list of current mirrors vs old mirrors, notify if one has appeared/disappeared + if ($mirror_list != $previous_mirror_list) + $notices[] = sprintf(gettext("List of mirrors changed. Old: (%s) New: (%s)"), implode(", ", $previous_mirror_list), implode(", ", $mirror_list)); + + // For each mirror, check the mirror status, notify if changed + foreach ($mirror_list as $mirror) { + if (is_array($previous_mirror_status[$mirror])) { + // Notify if the status changed + if ($mirror_status[$mirror]['status'] != $previous_mirror_status[$mirror]['status']) { + $notices[] = sprintf(gettext("Mirror %s status changed from %s to %s."), $mirror, $previous_mirror_status[$mirror]['status'], $mirror_status[$mirror]['status']); + } + // Notify if the drive count changed + if (count($mirror_status[$mirror]['components']) != count($previous_mirror_status[$mirror]['components'])) { + // Notify if the consumer count changed. + $notices[] = sprintf(gettext("Mirror %s consumer count changed from %d to %d."), $mirror, count($previous_mirror_status[$mirror]['components']), count($mirror_status[$mirror]['components'])); + } + if (strtoupper($mirror_status[$mirror]['status']) == "DEGRADED") { + // Check the drive status as it may be different. + asort($mirror_status[$mirror]['components']); + asort($previous_mirror_status[$mirror]['components']); + if ($mirror_status[$mirror]['components'] != $previous_mirror_status[$mirror]['components']) + $notices[] = sprintf(gettext("Mirror %s drive status changed. Old: (%s) New: (%s)"), + $mirror, + implode(", ", $previous_mirror_status[$mirror]['components']), + implode(", ", $mirror_status[$mirror]['components'])); + } + } + } + } +} +if (count($notices)) { + file_notice("gmirror", implode("\n ", $notices), "GEOM Mirror Status Change", 1); +} +// Write out current status if changed +if ($mirror_status != $previous_mirror_status) + file_put_contents($status_file, serialize($mirror_status)); + +?> \ No newline at end of file diff --git a/usr/local/www/diag_gmirror.php b/usr/local/www/diag_gmirror.php new file mode 100644 index 0000000000..3cb3b79f5c --- /dev/null +++ b/usr/local/www/diag_gmirror.php @@ -0,0 +1,344 @@ + + +"> + + + + gettext("Forget all formerly connected consumers"), + "clear" => gettext("Remove metadata from disk"), + "insert" => gettext("Insert consumer into mirror"), + "remove" => gettext("Remove consumer from mirror"), + "activate" => gettext("Reactivate consumer on mirror"), + "deactivate" => gettext("Deactivate consumer from mirror"), + "rebuild" => gettext("Force rebuild of mirror consumer"), +); + +/* User tried to pass a bogus action */ +if (!empty($_REQUEST['action']) && !array_key_exists($_REQUEST['action'], $action_list)) { + header("Location: diag_gmirror.php"); + return; +} + +if ($_POST) { + if (!isset($_POST['confirm']) || ($_POST['confirm'] != gettext("Confirm"))) { + header("Location: diag_gmirror.php"); + return; + } + $input_errors = ""; + + if (($_POST['action'] != "clear") && !is_valid_mirror($_POST['mirror'])) + $input_errors[] = gettext("You must supply a valid mirror name."); + + if (!empty($_POST['consumer']) && !is_valid_consumer($_POST['consumer'])) + $input_errors[] = gettext("You must supply a valid consumer name"); + + /* Additional action-specific validation that hasn't already been tested */ + switch ($_POST['action']) { + case "insert": + if (!is_consumer_unused($_POST['consumer'])) + $input_errors[] = gettext("Consumer is already in use and cannot be inserted. Remove consumer from existing mirror first."); + if (gmirror_consumer_has_metadata($_POST['consumer'])) + $input_errors[] = gettext("Consumer has metadata from an existing mirror. Clear metadata before inserting consumer."); + $mstat = gmirror_get_status_single($_POST['mirror']); + if (strtoupper($mstat) != "COMPLETE") + $input_errors[] = gettext("Mirror is not in a COMPLETE state, cannot insert consumer. Forget disconnected disks or wait for rebuild to finish."); + break; + case "clear": + if (!is_consumer_unused($_POST['consumer'])) + $input_errors[] = gettext("Consumer is in use and cannot be cleared. Deactivate disk first."); + if (!gmirror_consumer_has_metadata($_POST['consumer'])) + $input_errors[] = gettext("Consumer has no metadata to clear."); + break; + case "activate": + if (is_consumer_in_mirror($_POST['consumer'], $_POST['mirror'])) + $input_errors[] = gettext("Consumer is already present on specified mirror."); + if (!gmirror_consumer_has_metadata($_POST['consumer'])) + $input_errors[] = gettext("Consumer has no metadata and cannot be reactivated."); + + break; + case "remove": + case "deactivate": + case "rebuild": + if (!is_consumer_in_mirror($_POST['consumer'], $_POST['mirror'])) + $input_errors[] = gettext("Consumer must be present on the specified mirror."); + break; + } + +$result = 0; + if (empty($input_errors)) { + switch ($_POST['action']) { + case "forget": + $result = gmirror_forget_disconnected($_POST['mirror']); + break; + case "clear": + $result = gmirror_clear_consumer($_POST['consumer']); + break; + case "insert": + $result = gmirror_insert_consumer($_POST['mirror'], $_POST['consumer']); + break; + case "remove": + $result = gmirror_remove_consumer($_POST['mirror'], $_POST['consumer']); + break; + case "activate": + $result = gmirror_activate_consumer($_POST['mirror'], $_POST['consumer']); + break; + case "deactivate": + $result = gmirror_deactivate_consumer($_POST['mirror'], $_POST['consumer']); + break; + case "rebuild": + $result = gmirror_force_rebuild($_POST['mirror'], $_POST['consumer']); + break; + } + $redir = "Location: diag_gmirror.php"; + if ($result != 0) { + $redir .= "?error=" . urlencode($result); + } + /* If we reload the page too fast, the gmirror information may be missing or not up-to-date. */ + sleep(3); + header($redir); + return; + } +} + +$mirror_status = gmirror_get_status(); +$mirror_list = gmirror_get_mirrors(); +$unused_disks = gmirror_get_disks(); +$unused_consumers = array(); +foreach ($unused_disks as $disk) { + if (is_consumer_unused($disk)) + $unused_consumers = array_merge($unused_consumers, gmirror_get_all_unused_consumer_sizes_on_disk($disk)); +} + +if ($input_errors) + print_input_errors($input_errors); +if ($_GET["error"] && ($_GET["error"] != 0)) + print_info_box(gettext("There was an error performing the chosen mirror operation. Check the System Log for details.")); + +?> +
+ + + + +
+
+ + +   + + +
  + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
  + : +
+
: + " /> + +
: + " /> + + +
: + " /> + +
+
" /> +
+ + + + 0): ?> + + + + + + $name): + $components = count($name["components"]); ?> + + + + + + 1): + $morecomponents = array_slice($name["components"], 1); ?> + + + + + + + + + + + +
+ +
Size: + 		+ + +
[] + + 		+ + +
+ 1)): ?> + [] + [] + [] + +
+ + +
+ 1)): ?> + [] + [] + [] + +
+
+
+ + + + 0): ?> + + + + + + + + + + + + + + + + +
+ + + + [ ] +
[] + + + $mirror_size): ?> + + + + +
+
+
 
+

+
+
+ + + + +"; +echo "jQuery('#loading').html('');"; +echo ""; + +?> diff --git a/usr/local/www/fbegin.inc b/usr/local/www/fbegin.inc index 22cc90b9c6..680955eda5 100755 --- a/usr/local/www/fbegin.inc +++ b/usr/local/www/fbegin.inc @@ -210,6 +210,10 @@ $diagnostics_menu[] = array(gettext("Command Prompt"), "/exec.php"); $diagnostics_menu[] = array(gettext("DNS Lookup"), "/diag_dns.php"); $diagnostics_menu[] = array(gettext("Edit File"), "/edit.php"); $diagnostics_menu[] = array(gettext("Factory Defaults"), "/diag_defaults.php"); + +if(file_exists("/var/run/gmirror_active")) + $diagnostics_menu[] = array(gettext("GEOM Mirrors"), "/diag_gmirror.php" ); + $diagnostics_menu[] = array(gettext("Halt System"), "/halt.php" ); $diagnostics_menu[] = array(gettext("Limiter Info"), "/diag_limiter_info.php" ); $diagnostics_menu[] = array(gettext("NDP Table"), "/diag_ndp.php" ); diff --git a/usr/local/www/widgets/include/gmirror_status.inc b/usr/local/www/widgets/include/gmirror_status.inc deleted file mode 100644 index 8a04d4a4e3..0000000000 --- a/usr/local/www/widgets/include/gmirror_status.inc +++ /dev/null @@ -1,85 +0,0 @@ - 0) { - /* Loop through gmirror status output. */ - foreach ($status as $line) { - /* Split the line by whitespace */ - $all = preg_split("/[\s\t]+/", trim($line), 3); - if (count($all) == 3) { - /* If there are three items on a line, it is mirror name, status, and component */ - $currentmirror = $all[0]; - $mirrors[$currentmirror]["name"] = $all[0]; - $mirrors[$currentmirror]["status"] = $all[1]; - if (!is_array($mirrors[$currentmirror]["components"])) - $mirrors[$currentmirror]["components"] = array(); - $mirrors[$currentmirror]["components"][] = $all[2]; - } - } - } - /* Return an hash of mirrors and components */ - return $mirrors; -} - -function gmirror_html_status() { - $mirrors = gmirror_get_status(); - $output = ""; - if (count($mirrors) > 0) { - $output .= "\n"; - $output .= "Name\n"; - $output .= "Status\n"; - $output .= "Component\n"; - $output .= "\n"; - foreach ($mirrors as $mirror => $name) { - $components = count($name["components"]); - $output .= "\n"; - $output .= "{$name['name']}\n"; - $output .= "{$name['status']}\n"; - $output .= "{$name['components'][0]}\n"; - $output .= "\n"; - if (count($name["components"]) > 1) { - $morecomponents = array_slice($name["components"], 1); - foreach ($morecomponents as $component) { - $output .= "\n"; - $output .= "{$component}\n"; - $output .= "\n"; - } - } - } - } else { - $output .= "No Mirrors Found\n"; - } - // $output .= "Updated at " . date("F j, Y, g:i:s a") . "\n"; - return $output; -} -?> \ No newline at end of file diff --git a/usr/local/www/widgets/widgets/gmirror_status.widget.php b/usr/local/www/widgets/widgets/gmirror_status.widget.php index 543daf466c..2d040c00ca 100644 --- a/usr/local/www/widgets/widgets/gmirror_status.widget.php +++ b/usr/local/www/widgets/widgets/gmirror_status.widget.php @@ -28,7 +28,7 @@ $nocsrf = true; require_once("guiconfig.inc"); -require_once("/usr/local/www/widgets/include/gmirror_status.inc"); +require_once("gmirror.inc"); if ($_GET['textonly'] == "true") { header("Cache-Control: no-cache");