Implement library for lockfiles and use it in cfg code.

This is useful because the upcoming configuration database also needs a
lockfile implementation.

Also adds tests.

6 files changed, 329 insertions, 124 deletions

diff --git a/lib/automake.mk b/lib/automake.mk
index 9ba513ac..825395ac 100644
--- a/lib/automake.mk
+++ b/lib/automake.mk
@@ -55,6 +55,8 @@ lib_libopenvswitch_a_SOURCES = \
 	lib/learning-switch.h \
 	lib/list.c \
 	lib/list.h \
+	lib/lockfile.c \
+	lib/lockfile.h \
 	lib/mac-learning.c \
 	lib/mac-learning.h \
 	lib/netdev-linux.c \
@@ -175,9 +177,9 @@ install-data-local:
 
 # All the source files that have coverage counters.
 COVERAGE_FILES = \
-	lib/cfg.c \
 	lib/dpif.c \
 	lib/flow.c \
+	lib/lockfile.c \
 	lib/hmap.c \
 	lib/mac-learning.c \
 	lib/netdev.c \
diff --git a/lib/cfg.c b/lib/cfg.c
index 2cb4f345..d61cd778 100644
--- a/lib/cfg.c
+++ b/lib/cfg.c
@@ -25,10 +25,9 @@
 #include <netinet/in.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/stat.h>
-#include <unistd.h>
 #include "coverage.h"
 #include "dynamic-string.h"
+#include "lockfile.h"
 #include "ofpbuf.h"
 #include "packets.h"
 #include "svec.h"
@@ -52,8 +51,7 @@ static char *cfg_name;
 static char *tmp_name;
 
 /* Lock information. */
-static char *lock_name;
-static int lock_fd = -1;
+static struct lockfile *lockfile;
 
 /* Flag to indicate whether local modifications have been made. */
 static bool dirty;
@@ -106,15 +104,14 @@ cfg_init(void)
 int
 cfg_set_file(const char *file_name)
 {
-    const char *slash;
+    char *lock_name;
     int fd;
 
     if (cfg_name) {
-        assert(lock_fd < 0);
+        assert(!lockfile);
         free(cfg_name);
-        free(lock_name);
         free(tmp_name);
-        cfg_name = lock_name = tmp_name = NULL;
+        cfg_name = tmp_name = NULL;
     }
 
     /* Make sure that we can open this file for reading. */
@@ -131,18 +128,11 @@ cfg_set_file(const char *file_name)
      * rename(tmp_name, cfg_name) will work. */
     tmp_name = xasprintf("%s.~tmp~", file_name);
 
-    /* Put the lock file in the same directory as cfg_name, but prefixed by
-     * a dot so as not to garner administrator interest. */
-    slash = strrchr(file_name, '/');
-    if (slash) {
-        lock_name = xasprintf("%.*s/.%s.~lock~",
-                              slash - file_name, file_name, slash + 1);
-    } else {
-        lock_name = xasprintf(".%s.~lock~", file_name);
-    }
-
+    lock_name = lockfile_name(file_name);
     VLOG_INFO("using \"%s\" as configuration file, \"%s\" as lock file",
               file_name, lock_name);
+    free(lock_name);
+
     return 0;
 }
 
@@ -280,61 +270,12 @@ cfg_get_cookie(uint8_t *cookie)
 void
 cfg_unlock(void)
 {
-    if (lock_fd != -1) {
-        COVERAGE_INC(cfg_unlock);
-        close(lock_fd);
-        lock_fd = -1;
-    }
-}
-
-static int
-open_lockfile(const char *name)
-{
-    for (;;) {
-        /* Try to open an existing lock file. */
-        int fd = open(name, O_RDWR);
-        if (fd >= 0) {
-            return fd;
-        } else if (errno != ENOENT) {
-            VLOG_WARN("%s: failed to open lock file: %s",
-                      name, strerror(errno));
-            return -errno;
-        }
-
-        /* Try to create a new lock file. */
-        VLOG_INFO("%s: lock file does not exist, creating", name);
-        fd = open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
-        if (fd >= 0) {
-            return fd;
-        } else if (errno != EEXIST) {
-            VLOG_WARN("%s: failed to create lock file: %s",
-                      name, strerror(errno));
-            return -errno;
-        }
-
-        /* Someone else created the lock file.  Try again. */
+    if (lockfile) {
+        lockfile_unlock(lockfile);
+        lockfile = NULL;
     }
 }
 
-static int
-try_lock(int fd, bool block)
-{
-    struct flock l;
-    int error;
-
-    memset(&l, 0, sizeof l);
-    l.l_type = F_WRLCK;
-    l.l_whence = SEEK_SET;
-    l.l_start = 0;
-    l.l_len = 0;
-
-    time_disable_restart();
-    error = fcntl(fd, block ? F_SETLKW : F_SETLK, &l) == -1 ? errno : 0;
-    time_enable_restart();
-    
-    return error;
-}
-
 /* Locks the configuration file against modification by other processes and
  * re-reads it from disk.
  *
@@ -346,65 +287,18 @@ try_lock(int fd, bool block)
 int
 cfg_lock(uint8_t *cookie, int timeout)
 {
-    long long int start;
-    long long int elapsed = 0;
-    int fd;
-    uint8_t curr_cookie[CFG_COOKIE_LEN];
-
-    assert(lock_fd < 0);
-    COVERAGE_INC(cfg_lock);
-
-    time_refresh();
-    start = time_msec();
-    for (;;) {
-        int error;
-
-        /* Open lock file. */
-        fd = open_lockfile(lock_name);
-        if (fd < 0) {
-            return -fd;
-        }
-
-        /* Try to lock it.  This will block (if 'timeout' > 0). */
-        error = try_lock(fd, timeout > 0);
-        time_refresh();
-        elapsed = time_msec() - start;
-        if (!error) {
-            /* Success! */
-            break;
-        }
-
-        /* Lock failed.  Close the lock file and reopen it on the next
-         * iteration, just in case someone deletes it underneath us (even
-         * though that should not happen). */
-        close(fd);
-        if (error != EINTR) {
-            /* Hard error, give up. */
-            COVERAGE_INC(cfg_lock_error);
-            VLOG_WARN("%s: failed to lock file "
-                      "(after %lld ms, with %d-ms timeout): %s",
-                      lock_name, elapsed, timeout, strerror(error));
-            return error;
-        }
+    int error;
 
-        /* Probably, the periodic timer set up by time_init() woke up us.  Just
-         * check whether it's time to give up. */
-        if (timeout != INT_MAX && elapsed >= timeout) {
-            COVERAGE_INC(cfg_lock_timeout);
-            VLOG_WARN("%s: giving up on lock file after %lld ms",
-                      lock_name, elapsed);
-            return ETIMEDOUT;
-        }
-        COVERAGE_INC(cfg_lock_retry);
-    }
-    if (elapsed) {
-        VLOG_WARN("%s: waited %lld ms for lock file", lock_name, elapsed);
+    assert(!lockfile);
+    error = lockfile_lock(cfg_name, timeout, &lockfile);
+    if (error) {
+        return error;
     }
-    lock_fd = fd;
 
     cfg_read();
 
     if (cookie) {
+        uint8_t curr_cookie[CFG_COOKIE_LEN];
         cfg_get_cookie(curr_cookie);
 
         if (memcmp(curr_cookie, cookie, sizeof *curr_cookie)) {
diff --git a/lib/daemon.c b/lib/daemon.c
index a35c6393..e78538cb 100644
--- a/lib/daemon.c
+++ b/lib/daemon.c
@@ -23,6 +23,7 @@
 #include <unistd.h>
 #include "fatal-signal.h"
 #include "dirs.h"
+#include "lockfile.h"
 #include "timeval.h"
 #include "util.h"
 
@@ -224,6 +225,7 @@ daemonize(void)
                 chdir("/");
             }
             time_postfork();
+            lockfile_postfork();
             break;
 
         case -1:
diff --git a/lib/lockfile.c b/lib/lockfile.c
new file mode 100644
index 00000000..e5a041ee
--- /dev/null
+++ b/lib/lockfile.c
@@ -0,0 +1,280 @@
+ /* Copyright (c) 2008, 2009 Nicira Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+
+#include "lockfile.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "coverage.h"
+#include "hash.h"
+#include "hmap.h"
+#include "timeval.h"
+#include "util.h"
+
+#define THIS_MODULE VLM_lockfile
+#include "vlog.h"
+
+struct lockfile {
+    struct hmap_node hmap_node;
+    char *name;
+    dev_t device;
+    ino_t inode;
+    int fd;
+};
+
+/* Lock table.
+ *
+ * We have to do this stupid dance because POSIX says that closing *any* file
+ * descriptor for a file on which a process holds a lock drops *all* locks on
+ * that file.  That means that we can't afford to open a lockfile more than
+ * once. */
+static struct hmap lock_table = HMAP_INITIALIZER(&lock_table);
+
+static void lockfile_unhash(struct lockfile *);
+static int lockfile_try_lock(const char *name, bool block,
+                             struct lockfile **lockfilep);
+
+/* Returns the name of the lockfile that would be created for locking a file
+ * named 'file_name'.  The caller is responsible for freeing the returned
+ * name, with free(), when it is no longer needed. */
+char *
+lockfile_name(const char *file_name)
+{
+    const char *slash = strrchr(file_name, '/');
+    return (slash
+            ? xasprintf("%.*s/.%s.~lock~", slash - file_name, file_name,
+                        slash + 1)
+            : xasprintf(".%s.~lock~", file_name));
+}
+
+/* Locks the configuration file against modification by other processes and
+ * re-reads it from disk.
+ *
+ * The 'timeout' specifies the maximum number of milliseconds to wait for the
+ * config file to become free.  Use 0 to avoid waiting or INT_MAX to wait
+ * forever.
+ *
+ * Returns 0 on success, otherwise a positive errno value.  On success,
+ * '*lockfilep' is set to point to a new "struct lockfile *" that may be
+ * unlocked with lockfile_unlock().  On failure, '*lockfilep' is set to
+ * NULL. */
+int
+lockfile_lock(const char *file, int timeout, struct lockfile **lockfilep)
+{
+    /* Only exclusive ("write") locks are supported.  This is not a problem
+     * because the Open vSwitch code that currently uses lock files does so in
+     * stylized ways such that any number of readers may access a file while it
+     * is being written. */
+    long long int start, elapsed;
+    char *lock_name;
+    int error;
+
+    COVERAGE_INC(lockfile_lock);
+
+    lock_name = lockfile_name(file);
+    time_refresh();
+    start = time_msec();
+
+    do {
+        error = lockfile_try_lock(lock_name, timeout > 0, lockfilep);
+        time_refresh();
+        elapsed = time_msec() - start;
+    } while (error == EINTR && (timeout == INT_MAX || elapsed < timeout));
+
+    if (!error) {
+        if (elapsed) {
+            VLOG_WARN("%s: waited %lld ms for lock file",
+                      lock_name, elapsed);
+        }
+    } else if (error == EINTR) {
+        COVERAGE_INC(lockfile_timeout);
+        VLOG_WARN("%s: giving up on lock file after %lld ms",
+                  lock_name, elapsed);
+        error = ETIMEDOUT;
+    } else {
+        COVERAGE_INC(lockfile_error);
+        if (error == EACCES) {
+            error = EAGAIN;
+        }
+        VLOG_WARN("%s: failed to lock file "
+                  "(after %lld ms, with %d-ms timeout): %s",
+                  lock_name, elapsed, timeout, strerror(error));
+    }
+
+    free(lock_name);
+    return error;
+}
+
+/* Unlocks 'lockfile', which must have been created by a call to
+ * lockfile_lock(), and frees 'lockfile'. */
+void
+lockfile_unlock(struct lockfile *lockfile)
+{
+    if (lockfile) {
+        COVERAGE_INC(lockfile_unlock);
+        lockfile_unhash(lockfile);
+        free(lockfile->name);
+        free(lockfile);
+    }
+}
+
+/* Marks all the currently locked lockfiles as no longer locked.  It makes
+ * sense to call this function after fork(), because a child created by fork()
+ * does not hold its parents' locks. */
+void
+lockfile_postfork(void)
+{
+    struct lockfile *lockfile;
+
+    HMAP_FOR_EACH (lockfile, struct lockfile, hmap_node, &lock_table) {
+        if (lockfile->fd >= 0) {
+            VLOG_WARN("%s: child does not inherit lock", lockfile->name);
+            lockfile_unhash(lockfile);
+        }
+    }
+}
+
+static uint32_t
+lockfile_hash(dev_t device, ino_t inode)
+{
+    return hash_bytes(&device, sizeof device,
+                      hash_bytes(&inode, sizeof inode, 0));
+}
+
+static struct lockfile *
+lockfile_find(dev_t device, ino_t inode)
+{
+    struct lockfile *lockfile;
+
+    HMAP_FOR_EACH_WITH_HASH (lockfile, struct lockfile, hmap_node,
+                             lockfile_hash(device, inode), &lock_table) {
+        if (lockfile->device == device && lockfile->inode == inode) {
+            return lockfile;
+        }
+    }
+    return NULL;
+}
+
+static void
+lockfile_unhash(struct lockfile *lockfile)
+{
+    if (lockfile->fd >= 0) {
+        close(lockfile->fd);
+        lockfile->fd = -1;
+        hmap_remove(&lock_table, &lockfile->hmap_node);
+    }
+}
+
+static struct lockfile *
+lockfile_register(const char *name, dev_t device, ino_t inode, int fd)
+{
+    struct lockfile *lockfile;
+
+    lockfile = lockfile_find(device, inode);
+    if (lockfile) {
+        VLOG_ERR("%s: lock file disappeared and reappeared!", name);
+        lockfile_unhash(lockfile);
+    }
+
+    lockfile = xmalloc(sizeof *lockfile);
+    lockfile->name = xstrdup(name);
+    lockfile->device = device;
+    lockfile->inode = inode;
+    lockfile->fd = fd;
+    hmap_insert(&lock_table, &lockfile->hmap_node,
+                lockfile_hash(device, inode));
+    return lockfile;
+}
+
+static int
+lockfile_try_lock(const char *name, bool block, struct lockfile **lockfilep)
+{
+    struct flock l;
+    struct stat s;
+    int error;
+    int fd;
+
+    *lockfilep = NULL;
+
+    /* Open the lock file, first creating it if necessary. */
+    for (;;) {
+        /* Check whether we've already got a lock on that file. */
+        if (!stat(name, &s)) {
+            if (lockfile_find(s.st_dev, s.st_ino)) {
+                return EDEADLK;
+            }
+        } else if (errno != ENOENT) {
+            VLOG_WARN("%s: failed to stat lock file: %s",
+                      name, strerror(errno));
+            return errno;
+        }
+
+        /* Try to open an existing lock file. */
+        fd = open(name, O_RDWR);
+        if (fd >= 0) {
+            break;
+        } else if (errno != ENOENT) {
+            VLOG_WARN("%s: failed to open lock file: %s",
+                      name, strerror(errno));
+            return errno;
+        }
+
+        /* Try to create a new lock file. */
+        VLOG_INFO("%s: lock file does not exist, creating", name);
+        fd = open(name, O_RDWR | O_CREAT | O_EXCL, 0600);
+        if (fd >= 0) {
+            break;
+        } else if (errno != EEXIST) {
+            VLOG_WARN("%s: failed to create lock file: %s",
+                      name, strerror(errno));
+            return errno;
+        }
+
+        /* Someone else created the lock file.  Try again. */
+    }
+
+    /* Get the inode and device number for the lock table. */
+    if (fstat(fd, &s)) {
+        VLOG_ERR("%s: failed to fstat lock file: %s", name, strerror(errno));
+        close(fd);
+        return errno;
+    }
+
+    /* Try to lock the file. */
+    memset(&l, 0, sizeof l);
+    l.l_type = F_WRLCK;
+    l.l_whence = SEEK_SET;
+    l.l_start = 0;
+    l.l_len = 0;
+
+    time_disable_restart();
+    error = fcntl(fd, block ? F_SETLKW : F_SETLK, &l) == -1 ? errno : 0;
+    time_enable_restart();
+
+    if (!error) {
+        *lockfilep = lockfile_register(name, s.st_dev, s.st_ino, fd);
+    } else {
+        close(fd);
+    }
+    return error;
+}
+
diff --git a/lib/lockfile.h b/lib/lockfile.h
new file mode 100644
index 00000000..c52fa215
--- /dev/null
+++ b/lib/lockfile.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2008, 2009 Nicira Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LOCKFILE_H
+#define LOCKFILE_H 1
+
+struct lockfile;
+
+char *lockfile_name(const char *file);
+int lockfile_lock(const char *file, int timeout, struct lockfile **);
+void lockfile_unlock(struct lockfile *);
+void lockfile_postfork(void);
+
+#endif /* lib/lockfile.h */
diff --git a/lib/vlog-modules.def b/lib/vlog-modules.def
index ce298b55..59ab045c 100644
--- a/lib/vlog-modules.def
+++ b/lib/vlog-modules.def
@@ -42,6 +42,7 @@ VLOG_MODULE(flow)
 VLOG_MODULE(in_band)
 VLOG_MODULE(leak_checker)
 VLOG_MODULE(learning_switch)
+VLOG_MODULE(lockfile)
 VLOG_MODULE(mac_learning)
 VLOG_MODULE(mgmt)
 VLOG_MODULE(netdev)