Add syslog and fatal signal handler feature
authorBernd Schubert <bschubert@ddn.com>
Wed, 10 Jul 2024 21:04:46 +0000 (23:04 +0200)
committerBernd Schubert <bernd.schubert@fastmail.fm>
Sun, 14 Jul 2024 12:28:44 +0000 (14:28 +0200)
I see random ENOTCONN failures in xfstest generic/013 and generic/014
in my branch, but earliest on the 2nd run - takes ~12hours to get
the issue, but then there are no further information logged.
ENOTCONN points to a daemon crash - I need backtraces and a core dump.

This adds optional handling of fatal signals to print a core dump
and optional syslog logging with these new public functions:

fuse_set_fail_signal_handlers()
    In addition to the existing fuse_set_signal_handlers(). This is not
    enabled together with fuse_set_signal_handlers(), as it is change
    in behavior and file systems might already have their own fatal
    handlers.

fuse_log_enable_syslog
    Print logs to syslog instead of stderr

fuse_log_close_syslog
    Close syslog (for now just does closelog())

Code in fuse_signals.c is also updated, to be an array of signals,
and setting signal handlers is now down with a for-loop instead
of one hand coded set_one_signal_handler() per signal.

ChangeLog.rst
example/passthrough_hp.cc
include/fuse_common.h
include/fuse_log.h
lib/fuse_log.c
lib/fuse_signals.c
lib/fuse_versionscript

index cab3e81f45867acaccd63e3ed403c5c430ce971b..0ed7d782d48437fb9eea5c06821f9005b8c942ad 100644 (file)
@@ -1,3 +1,11 @@
+libfuse 3.17 (unreleased)
+========================
+
+* Allows to handle fatal signals and to print a backtrace.
+  New public function: fuse_set_fail_signal_handlers()
+* Allows fuse_log() messages to be send to syslog instead of stderr
+  New public functions: fuse_log_enable_syslog() and fuse_log_close_syslog()
+
 libfuse 3.16.2 (2023-10-10)
 ===========================
 
index e3c82250a3d19962d20e11a6e7445d919e728be9..708a38e5d697c170d95ac17ea85630949599c39b 100644 (file)
@@ -75,6 +75,7 @@
 #include <fstream>
 #include <thread>
 #include <iomanip>
+#include <syslog.h>
 
 using namespace std;
 
@@ -1436,6 +1437,9 @@ int main(int argc, char *argv[]) {
     if (fuse_set_signal_handlers(se) != 0)
         goto err_out2;
 
+    if (fuse_set_fail_signal_handlers(se) != 0)
+        goto err_out2;
+
     // Don't apply umask, use modes exactly as specified
     umask(0);
 
@@ -1452,12 +1456,14 @@ int main(int argc, char *argv[]) {
 
     fuse_daemonize(fs.foreground);
 
+    if (!fs.foreground)
+        fuse_log_enable_syslog("passthrough-hp", LOG_PID | LOG_CONS, LOG_DAEMON);
+
     if (options.count("single"))
         ret = fuse_session_loop(se);
     else
         ret = fuse_session_loop_mt(se, loop_config);
 
-
     fuse_session_unmount(se);
 
 err_out3:
@@ -1469,6 +1475,9 @@ err_out1:
     fuse_loop_cfg_destroy(loop_config);
     fuse_opt_free_args(&args);
 
+    if (!fs.foreground)
+        fuse_log_close_syslog();
+
     return ret ? 1 : 0;
 }
 
index 04ecb95876b2f55233ba32cb3b42f0e289c646fc..6de2640786cbca8c9253d7632ac41f93abb2f953 100644 (file)
@@ -954,6 +954,23 @@ ssize_t fuse_buf_copy(struct fuse_bufvec *dst, struct fuse_bufvec *src,
  */
 int fuse_set_signal_handlers(struct fuse_session *se);
 
+/**
+ * Print a stack backtrace diagnostic on critical signals ()
+ *
+ * Stores session in a global variable.         May only be called once per
+ * process until fuse_remove_signal_handlers() is called.
+ *
+ * Once either of the POSIX signals arrives, the signal handler calls
+ * fuse_session_exit().
+ *
+ * @param se the session to exit
+ * @return 0 on success, -1 on failure
+ *
+ * See also:
+ * fuse_remove_signal_handlers()
+ */
+int fuse_set_fail_signal_handlers(struct fuse_session *se);
+
 /**
  * Restore default signal handlers
  *
index 5e112e0f5343c7f8e2e75160c8e60bccd3b8037e..c8559572a6b3c4f4d7374efa4f748ecadd290393 100644 (file)
@@ -75,6 +75,18 @@ void fuse_set_log_func(fuse_log_func_t func);
  */
 void fuse_log(enum fuse_log_level level, const char *fmt, ...);
 
+/**
+ * Switch default log handler from stderr to syslog
+ *
+ * Passed options are according to 'man 3 openlog'
+ */
+void fuse_log_enable_syslog(const char *ident, int option, int facility);
+
+/**
+ * To be called at teardown to close syslog.
+*/
+void fuse_log_close_syslog(void);
+
 #ifdef __cplusplus
 }
 #endif
index 0d268ab014f5c863b6fdd76d30d0a6002c131d79..95d6379c01d54dc6cefd2a84a658b567798f7455 100644 (file)
 
 #include <stdarg.h>
 #include <stdio.h>
+#include <stdbool.h>
+#include <syslog.h>
 
-static void default_log_func(
-               __attribute__(( unused )) enum fuse_log_level level,
-               const char *fmt, va_list ap)
+#define MAX_SYSLOG_LINE_LEN 512
+
+static bool to_syslog = false;
+
+static void default_log_func(__attribute__((unused)) enum fuse_log_level level,
+                            const char *fmt, va_list ap)
 {
-       vfprintf(stderr, fmt, ap);
+       if (to_syslog) {
+               int sys_log_level;
+
+               /*
+                * with glibc fuse_log_level has identical values as
+                * syslog levels, but we also support BSD - better we convert to
+                * be sure.
+                */
+               switch (level) {
+               case FUSE_LOG_DEBUG:
+                       sys_log_level = LOG_DEBUG;
+                       break;
+               case FUSE_LOG_INFO:
+                       sys_log_level = LOG_INFO;
+                       break;
+               case FUSE_LOG_NOTICE:
+                       sys_log_level = LOG_NOTICE;
+                       break;
+               case FUSE_LOG_WARNING:
+                       sys_log_level = LOG_WARNING;
+                       break;
+               case FUSE_LOG_ERR:
+                       sys_log_level = LOG_ERR;
+                       break;
+               case FUSE_LOG_CRIT:
+                       sys_log_level = LOG_CRIT;
+                       break;
+               case FUSE_LOG_ALERT:
+                       sys_log_level = LOG_ALERT;
+                       break;
+               case FUSE_LOG_EMERG:
+                       sys_log_level = LOG_EMERG;
+               }
+
+               char log[MAX_SYSLOG_LINE_LEN];
+               vsnprintf(log, MAX_SYSLOG_LINE_LEN, fmt, ap);
+               syslog(sys_log_level, "%s", log);
+       } else {
+               vfprintf(stderr, fmt, ap);
+       }
 }
 
 static fuse_log_func_t log_func = default_log_func;
@@ -38,3 +82,15 @@ void fuse_log(enum fuse_log_level level, const char *fmt, ...)
        log_func(level, fmt, ap);
        va_end(ap);
 }
+
+void fuse_log_enable_syslog(const char *ident, int option, int facility)
+{
+       to_syslog = true;
+
+       openlog(ident, option, facility);
+}
+
+void fuse_log_close_syslog(void)
+{
+       closelog();
+}
\ No newline at end of file
index 4c1d1728388d24c84b70db8919e01f9bbe444996..0380c8255b082433e74e6a51eecb3191e3ba53bd 100644 (file)
 #include <signal.h>
 #include <stdlib.h>
 #include <execinfo.h>
+#include <errno.h>
 
+static int teardown_sigs[] = { SIGHUP, SIGINT, SIGTERM };
+static int ignore_sigs[] = { SIGPIPE};
+static int fail_sigs[] = { SIGILL, SIGTRAP, SIGABRT, SIGBUS, SIGFPE, SIGSEGV };
 static struct fuse_session *fuse_instance;
 
+#define BT_STACK_SZ (1024 * 1024)
+static void *backtrace_buffer[BT_STACK_SZ];
+
 static void dump_stack(void)
 {
+       fprintf(stderr, "%s:%d\n", __func__, __LINE__);
 #ifdef HAVE_BACKTRACE
-       const size_t backtrace_sz = 1024 * 1024;
-       void* backtrace_buffer[backtrace_sz];
+       char **strings;
+
+       int nptrs = backtrace(backtrace_buffer, BT_STACK_SZ);
+       strings = backtrace_symbols(backtrace_buffer, nptrs);
+
+       fprintf(stderr, "%s: nptrs=%d\n", __func__, nptrs);
+
+       if (strings == NULL) {
+               fuse_log(FUSE_LOG_ERR, "Failed to get backtrace symbols: %s\n",
+                        strerror(errno));
+               return;
+       }
 
-       int err_fd = fileno(stderr);
+       for (int idx = 0; idx < nptrs; idx++)
+               fuse_log(FUSE_LOG_ERR, "%s\n", strings[idx]);
 
-       int trace_len = backtrace(backtrace_buffer, backtrace_sz);
-       backtrace_symbols_fd(backtrace_buffer, trace_len, err_fd);
+       free(strings);
 #endif
 }
 
 static void exit_handler(int sig)
 {
-       if (fuse_instance) {
-               fuse_session_exit(fuse_instance);
-               if(sig <= 0) {
-                       fuse_log(FUSE_LOG_ERR, "assertion error: signal value <= 0\n");
-                       dump_stack();
-                       abort();
-               }
+       if (fuse_instance == NULL)
+               return;
+
+       fuse_session_exit(fuse_instance);
+
+       if (sig < 0) {
+               fuse_log(FUSE_LOG_ERR,
+                               "assertion error: signal value <= 0\n");
+               dump_stack();
+               abort();
                fuse_instance->error = sig;
        }
+
+       fuse_instance->error = sig;
 }
 
+static void exit_backtrace(int sig)
+{
+       if (fuse_instance == NULL)
+               return;
+
+       fuse_session_exit(fuse_instance);
+
+       fuse_remove_signal_handlers(fuse_instance);
+       fuse_log(FUSE_LOG_ERR, "Got signal: %d\n", sig);
+       dump_stack();
+       abort();
+}
+
+
 static void do_nothing(int sig)
 {
        (void) sig;
@@ -74,33 +111,88 @@ static int set_one_signal_handler(int sig, void (*handler)(int), int remove)
        return 0;
 }
 
+static int _fuse_set_signal_handlers(int signals[], int nr_signals,
+                                    void (*handler)(int))
+{
+       for (int idx = 0; idx < nr_signals; idx++) {
+               int signal = signals[idx];
+
+               /*
+                * If we used SIG_IGN instead of the do_nothing function,
+                * then we would be unable to tell if we set SIG_IGN (and
+                * thus should reset to SIG_DFL in fuse_remove_signal_handlers)
+                * or if it was already set to SIG_IGN (and should be left
+                * untouched.
+               */
+               if (set_one_signal_handler(signal, handler, 0) == -1) {
+                       fuse_log(FUSE_LOG_ERR,
+                                "Failed to install signal handler for sig %d\n",
+                                signal);
+                       return -1;
+               }
+       }
+
+       return 0;
+}
+
 int fuse_set_signal_handlers(struct fuse_session *se)
 {
-       /* If we used SIG_IGN instead of the do_nothing function,
-          then we would be unable to tell if we set SIG_IGN (and
-          thus should reset to SIG_DFL in fuse_remove_signal_handlers)
-          or if it was already set to SIG_IGN (and should be left
-          untouched. */
-       if (set_one_signal_handler(SIGHUP, exit_handler, 0) == -1 ||
-           set_one_signal_handler(SIGINT, exit_handler, 0) == -1 ||
-           set_one_signal_handler(SIGTERM, exit_handler, 0) == -1 ||
-           set_one_signal_handler(SIGPIPE, do_nothing, 0) == -1)
-               return -1;
+       size_t nr_signals;
+       int rc;
+
+       nr_signals = sizeof(teardown_sigs) / sizeof(teardown_sigs[0]);
+       rc = _fuse_set_signal_handlers(teardown_sigs, nr_signals, exit_handler);
+       if (rc < 0)
+               return rc;
 
-       fuse_instance = se;
+       nr_signals = sizeof(ignore_sigs) / sizeof(ignore_sigs[0]);
+       rc = _fuse_set_signal_handlers(ignore_sigs, nr_signals, do_nothing);
+       if (rc < 0)
+               return rc;
+
+       if (fuse_instance == NULL)
+               fuse_instance = se;
        return 0;
 }
 
+int fuse_set_fail_signal_handlers(struct fuse_session *se)
+{
+       size_t nr_signals = sizeof(fail_sigs) / sizeof(fail_sigs[0]);
+
+       int rc = _fuse_set_signal_handlers(fail_sigs, nr_signals,
+                                          exit_backtrace);
+       if (rc < 0)
+               return rc;
+
+       if (fuse_instance == NULL)
+               fuse_instance = se;
+
+       return 0;
+}
+
+static void _fuse_remove_signal_handlers(int signals[], int nr_signals,
+                                        void (*handler)(int))
+{
+       for (int idx = 0; idx < nr_signals; idx++)
+               set_one_signal_handler(signals[idx], handler, 1);
+}
+
 void fuse_remove_signal_handlers(struct fuse_session *se)
 {
+       size_t nr_signals;
+
        if (fuse_instance != se)
                fuse_log(FUSE_LOG_ERR,
                        "fuse: fuse_remove_signal_handlers: unknown session\n");
        else
                fuse_instance = NULL;
 
-       set_one_signal_handler(SIGHUP, exit_handler, 1);
-       set_one_signal_handler(SIGINT, exit_handler, 1);
-       set_one_signal_handler(SIGTERM, exit_handler, 1);
-       set_one_signal_handler(SIGPIPE, do_nothing, 1);
+       nr_signals = sizeof(teardown_sigs) / sizeof(teardown_sigs[0]);
+       _fuse_remove_signal_handlers(teardown_sigs, nr_signals, exit_handler);
+
+       nr_signals = sizeof(ignore_sigs) / sizeof(ignore_sigs[0]);
+       _fuse_remove_signal_handlers(ignore_sigs, nr_signals, do_nothing);
+
+       nr_signals = sizeof(fail_sigs) / sizeof(fail_sigs[0]);
+       _fuse_remove_signal_handlers(fail_sigs, nr_signals, exit_backtrace);
 }
index 10583728300dee2d2d834cb5b217e409c9ff52ab..14cbca142271f941175b0f1868caccb190ca371f 100644 (file)
@@ -198,6 +198,9 @@ FUSE_3.17 {
                fuse_passthrough_close;
                fuse_session_custom_io_30;
                fuse_session_custom_io_317;
+               fuse_set_fail_signal_handlers;
+               fuse_log_enable_syslog;
+               fuse_log_close_syslog;
 } FUSE_3.12;
 
 # Local Variables: