check bans and quiets for cmode -n/nonmember PRIVMSG

[solanum.git] / tests / runtests.c

/*
 * Run a set of tests, reporting results.
 *
 * Usage:
 *
 *      runtests [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>
 *      runtests [-hv] [-b <build-dir>] [-s <source-dir>] <test> [<test> ...]
 *      runtests -o [-h] [-b <build-dir>] [-s <source-dir>] <test>
 *
 * In the first case, expects a list of executables located in the given file,
 * one line per executable.  For each one, runs it as part of a test suite,
 * reporting results.  In the second case, use the same infrastructure, but
 * run only the tests listed on the command line.
 *
 * Test output should start with a line containing the number of tests
 * (numbered from 1 to this number), optionally preceded by "1..", although
 * that line may be given anywhere in the output.  Each additional line should
 * be in the following format:
 *
 *      ok <number>
 *      not ok <number>
 *      ok <number> # skip
 *      not ok <number> # todo
 *
 * where <number> is the number of the test.  An optional comment is permitted
 * after the number if preceded by whitespace.  ok indicates success, not ok
 * indicates failure.  "# skip" and "# todo" are a special cases of a comment,
 * and must start with exactly that formatting.  They indicate the test was
 * skipped for some reason (maybe because it doesn't apply to this platform)
 * or is testing something known to currently fail.  The text following either
 * "# skip" or "# todo" and whitespace is the reason.
 *
 * As a special case, the first line of the output may be in the form:
 *
 *      1..0 # skip some reason
 *
 * which indicates that this entire test case should be skipped and gives a
 * reason.
 *
 * Any other lines are ignored, although for compliance with the TAP protocol
 * all lines other than the ones in the above format should be sent to
 * standard error rather than standard output and start with #.
 *
 * This is a subset of TAP as documented in Test::Harness::TAP or
 * TAP::Parser::Grammar, which comes with Perl.
 *
 * If the -o option is given, instead run a single test and display all of its
 * output.  This is intended for use with failing tests so that the person
 * running the test suite can get more details about what failed.
 *
 * If built with the C preprocessor symbols C_TAP_SOURCE and C_TAP_BUILD
 * defined, C TAP Harness will export those values in the environment so that
 * tests can find the source and build directory and will look for tests under
 * both directories.  These paths can also be set with the -b and -s
 * command-line options, which will override anything set at build time.
 *
 * If the -v option is given, or the C_TAP_VERBOSE environment variable is set,
 * display the full output of each test as it runs rather than showing a
 * summary of the results of each test.
 *
 * Any bug reports, bug fixes, and improvements are very much welcome and
 * should be sent to the e-mail address below.  This program is part of C TAP
 * Harness <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
 *
 * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,
 *     2014, 2015, 2016 Russ Allbery <eagle@eyrie.org>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
*/

/* Required for fdopen(), getopt(), and putenv(). */
#if defined(__STRICT_ANSI__) || defined(PEDANTIC)
# ifndef _XOPEN_SOURCE
#  define _XOPEN_SOURCE 500
# endif
#endif

#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

/* sys/time.h must be included before sys/resource.h on some platforms. */
#include <sys/resource.h>

/* AIX 6.1 (and possibly later) doesn't have WCOREDUMP. */
#ifndef WCOREDUMP
# define WCOREDUMP(status) ((unsigned)(status) & 0x80)
#endif

/*
 * POSIX requires that these be defined in <unistd.h>, but they're not always
 * available.  If one of them has been defined, all the rest almost certainly
 * have.
 */
#ifndef STDIN_FILENO
# define STDIN_FILENO  0
# define STDOUT_FILENO 1
# define STDERR_FILENO 2
#endif

/*
 * Used for iterating through arrays.  Returns the number of elements in the
 * array (useful for a < upper bound in a for loop).
 */
#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))

/*
 * The source and build versions of the tests directory.  This is used to set
 * the C_TAP_SOURCE and C_TAP_BUILD environment variables (and the SOURCE and
 * BUILD environment variables set for backward compatibility) and find test
 * programs, if set.  Normally, this should be set as part of the build
 * process to the test subdirectories of $(abs_top_srcdir) and
 * $(abs_top_builddir) respectively.
 */
#ifndef C_TAP_SOURCE
# define C_TAP_SOURCE NULL
#endif
#ifndef C_TAP_BUILD
# define C_TAP_BUILD NULL
#endif

/* Test status codes. */
enum test_status {
    TEST_FAIL,
    TEST_PASS,
    TEST_SKIP,
    TEST_INVALID
};

/* Really, just a boolean, but this is more self-documenting. */
enum test_verbose {
    CONCISE = 0,
    VERBOSE = 1
};

/* Indicates the state of our plan. */
enum plan_status {
    PLAN_INIT,                  /* Nothing seen yet. */
    PLAN_FIRST,                 /* Plan seen before any tests. */
    PLAN_PENDING,               /* Test seen and no plan yet. */
    PLAN_FINAL                  /* Plan seen after some tests. */
};

/* Error exit statuses for test processes. */
#define CHILDERR_DUP    100     /* Couldn't redirect stderr or stdout. */
#define CHILDERR_EXEC   101     /* Couldn't exec child process. */
#define CHILDERR_STDIN  102     /* Couldn't open stdin file. */
#define CHILDERR_STDERR 103     /* Couldn't open stderr file. */

/* Structure to hold data for a set of tests. */
struct testset {
    char *file;                 /* The file name of the test. */
    char *path;                 /* The path to the test program. */
    enum plan_status plan;      /* The status of our plan. */
    unsigned long count;        /* Expected count of tests. */
    unsigned long current;      /* The last seen test number. */
    unsigned int length;        /* The length of the last status message. */
    unsigned long passed;       /* Count of passing tests. */
    unsigned long failed;       /* Count of failing lists. */
    unsigned long skipped;      /* Count of skipped tests (passed). */
    unsigned long allocated;    /* The size of the results table. */
    enum test_status *results;  /* Table of results by test number. */
    unsigned int aborted;       /* Whether the set was aborted. */
    int reported;               /* Whether the results were reported. */
    int status;                 /* The exit status of the test. */
    unsigned int all_skipped;   /* Whether all tests were skipped. */
    char *reason;               /* Why all tests were skipped. */
};

/* Structure to hold a linked list of test sets. */
struct testlist {
    struct testset *ts;
    struct testlist *next;
};

/*
 * Usage message.  Should be used as a printf format with four arguments: the
 * path to runtests, given three times, and the usage_description.  This is
 * split into variables to satisfy the pedantic ISO C90 limit on strings.
 */
static const char usage_message[] = "\
Usage: %s [-hv] [-b <build-dir>] [-s <source-dir>] <test> ...\n\
       %s [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>\n\
       %s -o [-h] [-b <build-dir>] [-s <source-dir>] <test>\n\
\n\
Options:\n\
    -b <build-dir>      Set the build directory to <build-dir>\n\
%s";
static const char usage_extra[] = "\
    -l <list>           Take the list of tests to run from <test-list>\n\
    -o                  Run a single test rather than a list of tests\n\
    -s <source-dir>     Set the source directory to <source-dir>\n\
    -v                  Show the full output of each test\n\
\n\
runtests normally runs each test listed on the command line.  With the -l\n\
option, it instead runs every test listed in a file.  With the -o option,\n\
it instead runs a single test and shows its complete output.\n";

/*
 * Header used for test output.  %s is replaced by the file name of the list
 * of tests.
 */
static const char banner[] = "\n\
Running all tests listed in %s.  If any tests fail, run the failing\n\
test program with runtests -o to see more details.\n\n";

/* Header for reports of failed tests. */
static const char header[] = "\n\
Failed Set                 Fail/Total (%) Skip Stat  Failing Tests\n\
-------------------------- -------------- ---- ----  ------------------------";

/* Include the file name and line number in malloc failures. */
#define xcalloc(n, size)      x_calloc((n), (size), __FILE__, __LINE__)
#define xmalloc(size)         x_malloc((size), __FILE__, __LINE__)
#define xstrdup(p)            x_strdup((p), __FILE__, __LINE__)
#define xreallocarray(p, n, size) \
    x_reallocarray((p), (n), (size), __FILE__, __LINE__)

/*
 * __attribute__ is available in gcc 2.5 and later, but only with gcc 2.7
 * could you use the __format__ form of the attributes, which is what we use
 * (to avoid confusion with other macros).
 */
#ifndef __attribute__
# if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 7)
#  define __attribute__(spec)   /* empty */
# endif
#endif

/*
 * We use __alloc_size__, but it was only available in fairly recent versions
 * of GCC.  Suppress warnings about the unknown attribute if GCC is too old.
 * We know that we're GCC at this point, so we can use the GCC variadic macro
 * extension, which will still work with versions of GCC too old to have C99
 * variadic macro support.
 */
#if !defined(__attribute__) && !defined(__alloc_size__)
# if defined(__GNUC__) && !defined(__clang__)
#  if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 3)
#   define __alloc_size__(spec, args...) /* empty */
#  endif
# endif
#endif

/*
 * LLVM and Clang pretend to be GCC but don't support all of the __attribute__
 * settings that GCC does.  For them, suppress warnings about unknown
 * attributes on declarations.  This unfortunately will affect the entire
 * compilation context, but there's no push and pop available.
 */
#if !defined(__attribute__) && (defined(__llvm__) || defined(__clang__))
# pragma GCC diagnostic ignored "-Wattributes"
#endif

/* Declare internal functions that benefit from compiler attributes. */
static void sysdie(const char *, ...)
    __attribute__((__nonnull__, __noreturn__, __format__(printf, 1, 2)));
static void *x_calloc(size_t, size_t, const char *, int)
    __attribute__((__alloc_size__(1, 2), __malloc__, __nonnull__));
static void *x_malloc(size_t, const char *, int)
    __attribute__((__alloc_size__(1), __malloc__, __nonnull__));
static void *x_reallocarray(void *, size_t, size_t, const char *, int)
    __attribute__((__alloc_size__(2, 3), __malloc__, __nonnull__(4)));
static char *x_strdup(const char *, const char *, int)
    __attribute__((__malloc__, __nonnull__));


/*
 * Report a fatal error, including the results of strerror, and exit.
 */
static void
sysdie(const char *format, ...)
{
    int oerrno;
    va_list args;

    oerrno = errno;
    fflush(stdout);
    fprintf(stderr, "runtests: ");
    va_start(args, format);
    vfprintf(stderr, format, args);
    va_end(args);
    fprintf(stderr, ": %s\n", strerror(oerrno));
    exit(1);
}


/*
 * Allocate zeroed memory, reporting a fatal error and exiting on failure.
 */
static void *
x_calloc(size_t n, size_t size, const char *file, int line)
{
    void *p;

    n = (n > 0) ? n : 1;
    size = (size > 0) ? size : 1;
    p = calloc(n, size);
    if (p == NULL)
        sysdie("failed to calloc %lu bytes at %s line %d",
               (unsigned long) size, file, line);
    return p;
}


/*
 * Allocate memory, reporting a fatal error and exiting on failure.
 */
static void *
x_malloc(size_t size, const char *file, int line)
{
    void *p;

    p = malloc(size);
    if (p == NULL)
        sysdie("failed to malloc %lu bytes at %s line %d",
               (unsigned long) size, file, line);
    return p;
}


/*
 * Reallocate memory, reporting a fatal error and exiting on failure.
 *
 * We should technically use SIZE_MAX here for the overflow check, but
 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
 * guarantee that it exists.  They do guarantee that UINT_MAX exists, and we
 * can assume that UINT_MAX <= SIZE_MAX.  And we should not be allocating
 * anything anywhere near that large.
 *
 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
 * I disbelieve in the existence of such systems and they will have to cope
 * without overflow checks.)
 */
static void *
x_reallocarray(void *p, size_t n, size_t size, const char *file, int line)
{
    if (n > 0 && UINT_MAX / n <= size)
        sysdie("realloc too large at %s line %d", file, line);
    p = realloc(p, n * size);
    if (p == NULL)
        sysdie("failed to realloc %lu bytes at %s line %d",
               (unsigned long) (n * size), file, line);
    return p;
}


/*
 * Copy a string, reporting a fatal error and exiting on failure.
 */
static char *
x_strdup(const char *s, const char *file, int line)
{
    char *p;
    size_t len;

    len = strlen(s) + 1;
    p = malloc(len);
    if (p == NULL)
        sysdie("failed to strdup %lu bytes at %s line %d",
               (unsigned long) len, file, line);
    memcpy(p, s, len);
    return p;
}


/*
 * Form a new string by concatenating multiple strings.  The arguments must be
 * terminated by (const char *) 0.
 *
 * This function only exists because we can't assume asprintf.  We can't
 * simulate asprintf with snprintf because we're only assuming SUSv3, which
 * does not require that snprintf with a NULL buffer return the required
 * length.  When those constraints are relaxed, this should be ripped out and
 * replaced with asprintf or a more trivial replacement with snprintf.
 */
static char *
concat(const char *first, ...)
{
    va_list args;
    char *result;
    const char *string;
    size_t offset;
    size_t length = 0;

    /*
     * Find the total memory required.  Ensure we don't overflow length.  We
     * aren't guaranteed to have SIZE_MAX, so use UINT_MAX as an acceptable
     * substitute (see the x_nrealloc comments).
     */
    va_start(args, first);
    for (string = first; string != NULL; string = va_arg(args, const char *)) {
        if (length >= UINT_MAX - strlen(string)) {
            errno = EINVAL;
            sysdie("strings too long in concat");
        }
        length += strlen(string);
    }
    va_end(args);
    length++;

    /* Create the string. */
    result = xmalloc(length);
    va_start(args, first);
    offset = 0;
    for (string = first; string != NULL; string = va_arg(args, const char *)) {
        memcpy(result + offset, string, strlen(string));
        offset += strlen(string);
    }
    va_end(args);
    result[offset] = '\0';
    return result;
}


/*
 * Given a struct timeval, return the number of seconds it represents as a
 * double.  Use difftime() to convert a time_t to a double.
 */
static double
tv_seconds(const struct timeval *tv)
{
    return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
}


/*
 * Given two struct timevals, return the difference in seconds.
 */
static double
tv_diff(const struct timeval *tv1, const struct timeval *tv0)
{
    return tv_seconds(tv1) - tv_seconds(tv0);
}


/*
 * Given two struct timevals, return the sum in seconds as a double.
 */
static double
tv_sum(const struct timeval *tv1, const struct timeval *tv2)
{
    return tv_seconds(tv1) + tv_seconds(tv2);
}


/*
 * Given a pointer to a string, skip any leading whitespace and return a
 * pointer to the first non-whitespace character.
 */
static const char *
skip_whitespace(const char *p)
{
    while (isspace((unsigned char)(*p)))
        p++;
    return p;
}


/*
 * Start a program, connecting its stdout to a pipe on our end and its stderr
 * to /dev/null, and storing the file descriptor to read from in the two
 * argument.  Returns the PID of the new process.  Errors are fatal.
 */
static pid_t
test_start(const char *path, int *fd)
{
    int fds[2], infd, errfd;
    pid_t child;

    /* Create a pipe used to capture the output from the test program. */
    if (pipe(fds) == -1) {
        puts("ABORTED");
        fflush(stdout);
        sysdie("can't create pipe");
    }

    /* Fork a child process, massage the file descriptors, and exec. */
    child = fork();
    switch (child) {
    case -1:
        puts("ABORTED");
        fflush(stdout);
        sysdie("can't fork");

    /* In the child.  Set up our standard output. */
    case 0:
        close(fds[0]);
        close(STDOUT_FILENO);
        if (dup2(fds[1], STDOUT_FILENO) < 0)
            _exit(CHILDERR_DUP);
        close(fds[1]);

        /* Point standard input at /dev/null. */
        close(STDIN_FILENO);
        infd = open("/dev/null", O_RDONLY);
        if (infd < 0)
            _exit(CHILDERR_STDIN);
        if (infd != STDIN_FILENO) {
            if (dup2(infd, STDIN_FILENO) < 0)
                _exit(CHILDERR_DUP);
            close(infd);
        }

        /* Point standard error at /dev/null. */
        close(STDERR_FILENO);
        errfd = open("/dev/null", O_WRONLY);
        if (errfd < 0)
            _exit(CHILDERR_STDERR);
        if (errfd != STDERR_FILENO) {
            if (dup2(errfd, STDERR_FILENO) < 0)
                _exit(CHILDERR_DUP);
            close(errfd);
        }

        /* Now, exec our process. */
        if (execl(path, path, (char *) 0) == -1)
            _exit(CHILDERR_EXEC);
        break;

    /* In parent.  Close the extra file descriptor. */
    default:
        close(fds[1]);
        break;
    }
    *fd = fds[0];
    return child;
}


/*
 * Back up over the output saying what test we were executing.
 */
static void
test_backspace(struct testset *ts)
{
    unsigned int i;

    if (!isatty(STDOUT_FILENO))
        return;
    for (i = 0; i < ts->length; i++)
        putchar('\b');
    for (i = 0; i < ts->length; i++)
        putchar(' ');
    for (i = 0; i < ts->length; i++)
        putchar('\b');
    ts->length = 0;
}


/*
 * Allocate or resize the array of test results to be large enough to contain
 * the test number in.
 */
static void
resize_results(struct testset *ts, unsigned long n)
{
    unsigned long i;
    size_t s;

    /* If there's already enough space, return quickly. */
    if (n <= ts->allocated)
        return;

    /*
     * If no space has been allocated, do the initial allocation.  Otherwise,
     * resize.  Start with 32 test cases and then add 1024 with each resize to
     * try to reduce the number of reallocations.
     */
    if (ts->allocated == 0) {
        s = (n > 32) ? n : 32;
        ts->results = xcalloc(s, sizeof(enum test_status));
    } else {
        s = (n > ts->allocated + 1024) ? n : ts->allocated + 1024;
        ts->results = xreallocarray(ts->results, s, sizeof(enum test_status));
    }

    /* Set the results for the newly-allocated test array. */
    for (i = ts->allocated; i < s; i++)
        ts->results[i] = TEST_INVALID;
    ts->allocated = s;
}


/*
 * Report an invalid test number and set the appropriate flags.  Pulled into a
 * separate function since we do this in several places.
 */
static void
invalid_test_number(struct testset *ts, long n, enum test_verbose verbose)
{
    if (!verbose)
        test_backspace(ts);
    printf("ABORTED (invalid test number %ld)\n", n);
    ts->aborted = 1;
    ts->reported = 1;
}


/*
 * Read the plan line of test output, which should contain the range of test
 * numbers.  We may initialize the testset structure here if we haven't yet
 * seen a test.  Return true if initialization succeeded and the test should
 * continue, false otherwise.
 */
static int
test_plan(const char *line, struct testset *ts, enum test_verbose verbose)
{
    long n;

    /*
     * Accept a plan without the leading 1.. for compatibility with older
     * versions of runtests.  This will only be allowed if we've not yet seen
     * a test result.
     */
    line = skip_whitespace(line);
    if (strncmp(line, "1..", 3) == 0)
        line += 3;

    /*
     * Get the count and check it for validity.
     *
     * If we have something of the form "1..0 # skip foo", the whole file was
     * skipped; record that.  If we do skip the whole file, zero out all of
     * our statistics, since they're no longer relevant.
     *
     * strtol is called with a second argument to advance the line pointer
     * past the count to make it simpler to detect the # skip case.
     */
    n = strtol(line, (char **) &line, 10);
    if (n == 0) {
        line = skip_whitespace(line);
        if (*line == '#') {
            line = skip_whitespace(line + 1);
            if (strncasecmp(line, "skip", 4) == 0) {
                line = skip_whitespace(line + 4);
                if (*line != '\0') {
                    ts->reason = xstrdup(line);
                    ts->reason[strlen(ts->reason) - 1] = '\0';
                }
                ts->all_skipped = 1;
                ts->aborted = 1;
                ts->count = 0;
                ts->passed = 0;
                ts->skipped = 0;
                ts->failed = 0;
                return 0;
            }
        }
    }
    if (n <= 0) {
        puts("ABORTED (invalid test count)");
        ts->aborted = 1;
        ts->reported = 1;
        return 0;
    }

    /*
     * If we are doing lazy planning, check the plan against the largest test
     * number that we saw and fail now if we saw a check outside the plan
     * range.
     */
    if (ts->plan == PLAN_PENDING && (unsigned long) n < ts->count) {
        invalid_test_number(ts, (long) ts->count, verbose);
        return 0;
    }

    /*
     * Otherwise, allocated or resize the results if needed and update count,
     * and then record that we've seen a plan.
     */
    resize_results(ts, (unsigned long) n);
    ts->count = (unsigned long) n;
    if (ts->plan == PLAN_INIT)
        ts->plan = PLAN_FIRST;
    else if (ts->plan == PLAN_PENDING)
        ts->plan = PLAN_FINAL;
    return 1;
}


/*
 * Given a single line of output from a test, parse it and return the success
 * status of that test.  Anything printed to stdout not matching the form
 * /^(not )?ok \d+/ is ignored.  Sets ts->current to the test number that just
 * reported status.
 */
static void
test_checkline(const char *line, struct testset *ts,
               enum test_verbose verbose)
{
    enum test_status status = TEST_PASS;
    const char *bail;
    char *end;
    long number;
    unsigned long current;
    int outlen;

    /* Before anything, check for a test abort. */
    bail = strstr(line, "Bail out!");
    if (bail != NULL) {
        bail = skip_whitespace(bail + strlen("Bail out!"));
        if (*bail != '\0') {
            size_t length;

            length = strlen(bail);
            if (bail[length - 1] == '\n')
                length--;
            if (!verbose)
                test_backspace(ts);
            printf("ABORTED (%.*s)\n", (int) length, bail);
            ts->reported = 1;
        }
        ts->aborted = 1;
        return;
    }

    /*
     * If the given line isn't newline-terminated, it was too big for an
     * fgets(), which means ignore it.
     */
    if (line[strlen(line) - 1] != '\n')
        return;

    /* If the line begins with a hash mark, ignore it. */
    if (line[0] == '#')
        return;

    /* If we haven't yet seen a plan, look for one. */
    if (ts->plan == PLAN_INIT && isdigit((unsigned char)(*line))) {
        if (!test_plan(line, ts, verbose))
            return;
    } else if (strncmp(line, "1..", 3) == 0) {
        if (ts->plan == PLAN_PENDING) {
            if (!test_plan(line, ts, verbose))
                return;
        } else {
            if (!verbose)
                test_backspace(ts);
            puts("ABORTED (multiple plans)");
            ts->aborted = 1;
            ts->reported = 1;
            return;
        }
    }

    /* Parse the line, ignoring something we can't parse. */
    if (strncmp(line, "not ", 4) == 0) {
        status = TEST_FAIL;
        line += 4;
    }
    if (strncmp(line, "ok", 2) != 0)
        return;
    line = skip_whitespace(line + 2);
    errno = 0;
    number = strtol(line, &end, 10);
    if (errno != 0 || end == line)
        current = ts->current + 1;
    else if (number <= 0) {
        invalid_test_number(ts, number, verbose);
        return;
    } else
        current = (unsigned long) number;
    if (current > ts->count && ts->plan == PLAN_FIRST) {
        invalid_test_number(ts, (long) current, verbose);
        return;
    }

    /* We have a valid test result.  Tweak the results array if needed. */
    if (ts->plan == PLAN_INIT || ts->plan == PLAN_PENDING) {
        ts->plan = PLAN_PENDING;
        resize_results(ts, current);
        if (current > ts->count)
            ts->count = current;
    }

    /*
     * Handle directives.  We should probably do something more interesting
     * with unexpected passes of todo tests.
     */
    while (isdigit((unsigned char)(*line)))
        line++;
    line = skip_whitespace(line);
    if (*line == '#') {
        line = skip_whitespace(line + 1);
        if (strncasecmp(line, "skip", 4) == 0)
            status = TEST_SKIP;
        if (strncasecmp(line, "todo", 4) == 0)
            status = (status == TEST_FAIL) ? TEST_SKIP : TEST_FAIL;
    }

    /* Make sure that the test number is in range and not a duplicate. */
    if (ts->results[current - 1] != TEST_INVALID) {
        if (!verbose)
            test_backspace(ts);
        printf("ABORTED (duplicate test number %lu)\n", current);
        ts->aborted = 1;
        ts->reported = 1;
        return;
    }

    /* Good results.  Increment our various counters. */
    switch (status) {
        case TEST_PASS: ts->passed++;   break;
        case TEST_FAIL: ts->failed++;   break;
        case TEST_SKIP: ts->skipped++;  break;
        case TEST_INVALID:              break;
    }
    ts->current = current;
    ts->results[current - 1] = status;
    if (!verbose && isatty(STDOUT_FILENO)) {
        test_backspace(ts);
        if (ts->plan == PLAN_PENDING)
            outlen = printf("%lu/?", current);
        else
            outlen = printf("%lu/%lu", current, ts->count);
        ts->length = (outlen >= 0) ? (unsigned int) outlen : 0;
        fflush(stdout);
    }
}


/*
 * Print out a range of test numbers, returning the number of characters it
 * took up.  Takes the first number, the last number, the number of characters
 * already printed on the line, and the limit of number of characters the line
 * can hold.  Add a comma and a space before the range if chars indicates that
 * something has already been printed on the line, and print ... instead if
 * chars plus the space needed would go over the limit (use a limit of 0 to
 * disable this).
 */
static unsigned int
test_print_range(unsigned long first, unsigned long last, unsigned long chars,
                 unsigned int limit)
{
    unsigned int needed = 0;
    unsigned long n;

    for (n = first; n > 0; n /= 10)
        needed++;
    if (last > first) {
        for (n = last; n > 0; n /= 10)
            needed++;
        needed++;
    }
    if (chars > 0)
        needed += 2;
    if (limit > 0 && chars + needed > limit) {
        needed = 0;
        if (chars <= limit) {
            if (chars > 0) {
                printf(", ");
                needed += 2;
            }
            printf("...");
            needed += 3;
        }
    } else {
        if (chars > 0)
            printf(", ");
        if (last > first)
            printf("%lu-", first);
        printf("%lu", last);
    }
    return needed;
}


/*
 * Summarize a single test set.  The second argument is 0 if the set exited
 * cleanly, a positive integer representing the exit status if it exited
 * with a non-zero status, and a negative integer representing the signal
 * that terminated it if it was killed by a signal.
 */
static void
test_summarize(struct testset *ts, int status)
{
    unsigned long i;
    unsigned long missing = 0;
    unsigned long failed = 0;
    unsigned long first = 0;
    unsigned long last = 0;

    if (ts->aborted) {
        fputs("ABORTED", stdout);
        if (ts->count > 0)
            printf(" (passed %lu/%lu)", ts->passed, ts->count - ts->skipped);
    } else {
        for (i = 0; i < ts->count; i++) {
            if (ts->results[i] == TEST_INVALID) {
                if (missing == 0)
                    fputs("MISSED ", stdout);
                if (first && i == last)
                    last = i + 1;
                else {
                    if (first)
                        test_print_range(first, last, missing - 1, 0);
                    missing++;
                    first = i + 1;
                    last = i + 1;
                }
            }
        }
        if (first)
            test_print_range(first, last, missing - 1, 0);
        first = 0;
        last = 0;
        for (i = 0; i < ts->count; i++) {
            if (ts->results[i] == TEST_FAIL) {
                if (missing && !failed)
                    fputs("; ", stdout);
                if (failed == 0)
                    fputs("FAILED ", stdout);
                if (first && i == last)
                    last = i + 1;
                else {
                    if (first)
                        test_print_range(first, last, failed - 1, 0);
                    failed++;
                    first = i + 1;
                    last = i + 1;
                }
            }
        }
        if (first)
            test_print_range(first, last, failed - 1, 0);
        if (!missing && !failed) {
            fputs(!status ? "ok" : "dubious", stdout);
            if (ts->skipped > 0) {
                if (ts->skipped == 1)
                    printf(" (skipped %lu test)", ts->skipped);
                else
                    printf(" (skipped %lu tests)", ts->skipped);
            }
        }
    }
    if (status > 0)
        printf(" (exit status %d)", status);
    else if (status < 0)
        printf(" (killed by signal %d%s)", -status,
               WCOREDUMP(ts->status) ? ", core dumped" : "");
    putchar('\n');
}


/*
 * Given a test set, analyze the results, classify the exit status, handle a
 * few special error messages, and then pass it along to test_summarize() for
 * the regular output.  Returns true if the test set ran successfully and all
 * tests passed or were skipped, false otherwise.
 */
static int
test_analyze(struct testset *ts)
{
    if (ts->reported)
        return 0;
    if (ts->all_skipped) {
        if (ts->reason == NULL)
            puts("skipped");
        else
            printf("skipped (%s)\n", ts->reason);
        return 1;
    } else if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
        switch (WEXITSTATUS(ts->status)) {
        case CHILDERR_DUP:
            if (!ts->reported)
                puts("ABORTED (can't dup file descriptors)");
            break;
        case CHILDERR_EXEC:
            if (!ts->reported)
                puts("ABORTED (execution failed -- not found?)");
            break;
        case CHILDERR_STDIN:
        case CHILDERR_STDERR:
            if (!ts->reported)
                puts("ABORTED (can't open /dev/null)");
            break;
        default:
            test_summarize(ts, WEXITSTATUS(ts->status));
            break;
        }
        return 0;
    } else if (WIFSIGNALED(ts->status)) {
        test_summarize(ts, -WTERMSIG(ts->status));
        return 0;
    } else if (ts->plan != PLAN_FIRST && ts->plan != PLAN_FINAL) {
        puts("ABORTED (no valid test plan)");
        ts->aborted = 1;
        return 0;
    } else {
        test_summarize(ts, 0);
        return (ts->failed == 0);
    }
}


/*
 * Runs a single test set, accumulating and then reporting the results.
 * Returns true if the test set was successfully run and all tests passed,
 * false otherwise.
 */
static int
test_run(struct testset *ts, enum test_verbose verbose)
{
    pid_t testpid, child;
    int outfd, status;
    unsigned long i;
    FILE *output;
    char buffer[BUFSIZ];

    /* Run the test program. */
    testpid = test_start(ts->path, &outfd);
    output = fdopen(outfd, "r");
    if (!output) {
        puts("ABORTED");
        fflush(stdout);
        sysdie("fdopen failed");
    }

    /*
     * Pass each line of output to test_checkline(), and print the line if
     * verbosity is requested.
     */
    while (!ts->aborted && fgets(buffer, sizeof(buffer), output)) {
        if (verbose)
            printf("%s", buffer);
        test_checkline(buffer, ts, verbose);
    }
    if (ferror(output) || ts->plan == PLAN_INIT)
        ts->aborted = 1;
    if (!verbose)
        test_backspace(ts);

    /*
     * Consume the rest of the test output, close the output descriptor,
     * retrieve the exit status, and pass that information to test_analyze()
     * for eventual output.
     */
    while (fgets(buffer, sizeof(buffer), output))
        if (verbose)
            printf("%s", buffer);
    fclose(output);
    child = waitpid(testpid, &ts->status, 0);
    if (child == (pid_t) -1) {
        if (!ts->reported) {
            puts("ABORTED");
            fflush(stdout);
        }
        sysdie("waitpid for %u failed", (unsigned int) testpid);
    }
    if (ts->all_skipped)
        ts->aborted = 0;
    status = test_analyze(ts);

    /* Convert missing tests to failed tests. */
    for (i = 0; i < ts->count; i++) {
        if (ts->results[i] == TEST_INVALID) {
            ts->failed++;
            ts->results[i] = TEST_FAIL;
            status = 0;
        }
    }
    return status;
}


/* Summarize a list of test failures. */
static void
test_fail_summary(const struct testlist *fails)
{
    struct testset *ts;
    unsigned int chars;
    unsigned long i, first, last, total;

    puts(header);

    /* Failed Set                 Fail/Total (%) Skip Stat  Failing (25)
       -------------------------- -------------- ---- ----  -------------- */
    for (; fails; fails = fails->next) {
        ts = fails->ts;
        total = ts->count - ts->skipped;
        printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts->file, ts->failed,
               total, total ? (ts->failed * 100.0) / total : 0,
               ts->skipped);
        if (WIFEXITED(ts->status))
            printf("%4d  ", WEXITSTATUS(ts->status));
        else
            printf("  --  ");
        if (ts->aborted) {
            puts("aborted");
            continue;
        }
        chars = 0;
        first = 0;
        last = 0;
        for (i = 0; i < ts->count; i++) {
            if (ts->results[i] == TEST_FAIL) {
                if (first != 0 && i == last)
                    last = i + 1;
                else {
                    if (first != 0)
                        chars += test_print_range(first, last, chars, 19);
                    first = i + 1;
                    last = i + 1;
                }
            }
        }
        if (first != 0)
            test_print_range(first, last, chars, 19);
        putchar('\n');
    }
}


/*
 * Check whether a given file path is a valid test.  Currently, this checks
 * whether it is executable and is a regular file.  Returns true or false.
 */
static int
is_valid_test(const char *path)
{
    struct stat st;

    if (access(path, X_OK) < 0)
        return 0;
    if (stat(path, &st) < 0)
        return 0;
    if (!S_ISREG(st.st_mode))
        return 0;
    return 1;
}


/*
 * Given the name of a test, a pointer to the testset struct, and the source
 * and build directories, find the test.  We try first relative to the current
 * directory, then in the build directory (if not NULL), then in the source
 * directory.  In each of those directories, we first try a "-t" extension and
 * then a ".t" extension.  When we find an executable program, we return the
 * path to that program.  If none of those paths are executable, just fill in
 * the name of the test as is.
 *
 * The caller is responsible for freeing the path member of the testset
 * struct.
 */
static char *
find_test(const char *name, const char *source, const char *build)
{
    char *path = NULL;
    const char *bases[3], *suffix, *base;
    unsigned int i, j;
    const char *suffixes[3] = { "-t", ".t", "" };

    /* Possible base directories. */
    bases[0] = ".";
    bases[1] = build;
    bases[2] = source;

    /* Try each suffix with each base. */
    for (i = 0; i < ARRAY_SIZE(suffixes); i++) {
        suffix = suffixes[i];
        for (j = 0; j < ARRAY_SIZE(bases); j++) {
            base = bases[j];
            if (base == NULL)
                continue;
            path = concat(base, "/", name, suffix, (const char *) 0);
            if (is_valid_test(path))
                return path;
            free(path);
            path = NULL;
        }
    }
    if (path == NULL)
        path = xstrdup(name);
    return path;
}


/*
 * Read a list of tests from a file, returning the list of tests as a struct
 * testlist, or NULL if there were no tests (such as a file containing only
 * comments).  Reports an error to standard error and exits if the list of
 * tests cannot be read.
 */
static struct testlist *
read_test_list(const char *filename)
{
    FILE *file;
    unsigned int line;
    size_t length;
    char buffer[BUFSIZ];
    const char *testname;
    struct testlist *listhead, *current;

    /* Create the initial container list that will hold our results. */
    listhead = xcalloc(1, sizeof(struct testlist));
    current = NULL;

    /*
     * Open our file of tests to run and read it line by line, creating a new
     * struct testlist and struct testset for each line.
     */
    file = fopen(filename, "r");
    if (file == NULL)
        sysdie("can't open %s", filename);
    line = 0;
    while (fgets(buffer, sizeof(buffer), file)) {
        line++;
        length = strlen(buffer) - 1;
        if (buffer[length] != '\n') {
            fprintf(stderr, "%s:%u: line too long\n", filename, line);
            exit(1);
        }
        buffer[length] = '\0';

        /* Skip comments, leading spaces, and blank lines. */
        testname = skip_whitespace(buffer);
        if (strlen(testname) == 0)
            continue;
        if (testname[0] == '#')
            continue;

        /* Allocate the new testset structure. */
        if (current == NULL)
            current = listhead;
        else {
            current->next = xcalloc(1, sizeof(struct testlist));
            current = current->next;
        }
        current->ts = xcalloc(1, sizeof(struct testset));
        current->ts->plan = PLAN_INIT;
        current->ts->file = xstrdup(testname);
    }
    fclose(file);

    /* If there were no tests, current is still NULL. */
    if (current == NULL) {
        free(listhead);
        return NULL;
    }

    /* Return the results. */
    return listhead;
}


/*
 * Build a list of tests from command line arguments.  Takes the argv and argc
 * representing the command line arguments and returns a newly allocated test
 * list, or NULL if there were no tests.  The caller is responsible for
 * freeing.
 */
static struct testlist *
build_test_list(char *argv[], int argc)
{
    int i;
    struct testlist *listhead, *current;

    /* Create the initial container list that will hold our results. */
    listhead = xcalloc(1, sizeof(struct testlist));
    current = NULL;

    /* Walk the list of arguments and create test sets for them. */
    for (i = 0; i < argc; i++) {
        if (current == NULL)
            current = listhead;
        else {
            current->next = xcalloc(1, sizeof(struct testlist));
            current = current->next;
        }
        current->ts = xcalloc(1, sizeof(struct testset));
        current->ts->plan = PLAN_INIT;
        current->ts->file = xstrdup(argv[i]);
    }

    /* If there were no tests, current is still NULL. */
    if (current == NULL) {
        free(listhead);
        return NULL;
    }

    /* Return the results. */
    return listhead;
}


/* Free a struct testset. */
static void
free_testset(struct testset *ts)
{
    free(ts->file);
    free(ts->path);
    free(ts->results);
    free(ts->reason);
    free(ts);
}


/*
 * Run a batch of tests.  Takes two additional parameters: the root of the
 * source directory and the root of the build directory.  Test programs will
 * be first searched for in the current directory, then the build directory,
 * then the source directory.  Returns true iff all tests passed, and always
 * frees the test list that's passed in.
 */
static int
test_batch(struct testlist *tests, const char *source, const char *build,
           enum test_verbose verbose)
{
    size_t length, i;
    size_t longest = 0;
    unsigned int count = 0;
    struct testset *ts;
    struct timeval start, end;
    struct rusage stats;
    struct testlist *failhead = NULL;
    struct testlist *failtail = NULL;
    struct testlist *current, *next;
    int succeeded;
    unsigned long total = 0;
    unsigned long passed = 0;
    unsigned long skipped = 0;
    unsigned long failed = 0;
    unsigned long aborted = 0;

    /* Walk the list of tests to find the longest name. */
    for (current = tests; current != NULL; current = current->next) {
        length = strlen(current->ts->file);
        if (length > longest)
            longest = length;
    }

    /*
     * Add two to longest and round up to the nearest tab stop.  This is how
     * wide the column for printing the current test name will be.
     */
    longest += 2;
    if (longest % 8)
        longest += 8 - (longest % 8);

    /* Start the wall clock timer. */
    gettimeofday(&start, NULL);

    /* Now, plow through our tests again, running each one. */
    for (current = tests; current != NULL; current = current->next) {
        ts = current->ts;

        /* Print out the name of the test file. */
        fputs(ts->file, stdout);
        if (verbose)
            fputs("\n\n", stdout);
        else
            for (i = strlen(ts->file); i < longest; i++)
                putchar('.');
        if (isatty(STDOUT_FILENO))
            fflush(stdout);

        /* Run the test. */
        ts->path = find_test(ts->file, source, build);
        succeeded = test_run(ts, verbose);
        fflush(stdout);
        if (verbose)
            putchar('\n');

        /* Record cumulative statistics. */
        aborted += ts->aborted;
        total += ts->count + ts->all_skipped;
        passed += ts->passed;
        skipped += ts->skipped + ts->all_skipped;
        failed += ts->failed;
        count++;

        /* If the test fails, we shuffle it over to the fail list. */
        if (!succeeded) {
            if (failhead == NULL) {
                failhead = xmalloc(sizeof(struct testset));
                failtail = failhead;
            } else {
                failtail->next = xmalloc(sizeof(struct testset));
                failtail = failtail->next;
            }
            failtail->ts = ts;
            failtail->next = NULL;
        }
    }
    total -= skipped;

    /* Stop the timer and get our child resource statistics. */
    gettimeofday(&end, NULL);
    getrusage(RUSAGE_CHILDREN, &stats);

    /* Summarize the failures and free the failure list. */
    if (failhead != NULL) {
        test_fail_summary(failhead);
        while (failhead != NULL) {
            next = failhead->next;
            free(failhead);
            failhead = next;
        }
    }

    /* Free the memory used by the test lists. */
    while (tests != NULL) {
        next = tests->next;
        free_testset(tests->ts);
        free(tests);
        tests = next;
    }

    /* Print out the final test summary. */
    putchar('\n');
    if (aborted != 0) {
        if (aborted == 1)
            printf("Aborted %lu test set", aborted);
        else
            printf("Aborted %lu test sets", aborted);
        printf(", passed %lu/%lu tests", passed, total);
    }
    else if (failed == 0)
        fputs("All tests successful", stdout);
    else
        printf("Failed %lu/%lu tests, %.2f%% okay", failed, total,
               (total - failed) * 100.0 / total);
    if (skipped != 0) {
        if (skipped == 1)
            printf(", %lu test skipped", skipped);
        else
            printf(", %lu tests skipped", skipped);
    }
    puts(".");
    printf("Files=%u,  Tests=%lu", count, total);
    printf(",  %.2f seconds", tv_diff(&end, &start));
    printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
           tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
           tv_sum(&stats.ru_utime, &stats.ru_stime));
    return (failed == 0 && aborted == 0);
}


/*
 * Run a single test case.  This involves just running the test program after
 * having done the environment setup and finding the test program.
 */
static void
test_single(const char *program, const char *source, const char *build)
{
    char *path;

    path = find_test(program, source, build);
    if (execl(path, path, (char *) 0) == -1)
        sysdie("cannot exec %s", path);
}


/*
 * Main routine.  Set the C_TAP_SOURCE, C_TAP_BUILD, SOURCE, and BUILD
 * environment variables and then, given a file listing tests, run each test
 * listed.
 */
int
main(int argc, char *argv[])
{
    int option;
    int status = 0;
    int single = 0;
    enum test_verbose verbose = CONCISE;
    char *c_tap_source_env = NULL;
    char *c_tap_build_env = NULL;
    char *source_env = NULL;
    char *build_env = NULL;
    const char *program;
    const char *shortlist;
    const char *list = NULL;
    const char *source = C_TAP_SOURCE;
    const char *build = C_TAP_BUILD;
    struct testlist *tests;

    program = argv[0];
    while ((option = getopt(argc, argv, "b:hl:os:v")) != EOF) {
        switch (option) {
        case 'b':
            build = optarg;
            break;
        case 'h':
            printf(usage_message, program, program, program, usage_extra);
            exit(0);
        case 'l':
            list = optarg;
            break;
        case 'o':
            single = 1;
            break;
        case 's':
            source = optarg;
            break;
        case 'v':
            verbose = VERBOSE;
            break;
        default:
            exit(1);
        }
    }
    argv += optind;
    argc -= optind;
    if ((list == NULL && argc < 1) || (list != NULL && argc > 0)) {
        fprintf(stderr, usage_message, program, program, program, usage_extra);
        exit(1);
    }

    /*
     * If C_TAP_VERBOSE is set in the environment, that also turns on verbose
     * mode.
     */
    if (getenv("C_TAP_VERBOSE") != NULL)
        verbose = VERBOSE;

    /*
     * Set C_TAP_SOURCE and C_TAP_BUILD environment variables.  Also set
     * SOURCE and BUILD for backward compatibility, although we're trying to
     * migrate to the ones with a C_TAP_* prefix.
     */
    if (source != NULL) {
        c_tap_source_env = concat("C_TAP_SOURCE=", source, (const char *) 0);
        if (putenv(c_tap_source_env) != 0)
            sysdie("cannot set C_TAP_SOURCE in the environment");
        source_env = concat("SOURCE=", source, (const char *) 0);
        if (putenv(source_env) != 0)
            sysdie("cannot set SOURCE in the environment");
    }
    if (build != NULL) {
        c_tap_build_env = concat("C_TAP_BUILD=", build, (const char *) 0);
        if (putenv(c_tap_build_env) != 0)
            sysdie("cannot set C_TAP_BUILD in the environment");
        build_env = concat("BUILD=", build, (const char *) 0);
        if (putenv(build_env) != 0)
            sysdie("cannot set BUILD in the environment");
    }

    /* Run the tests as instructed. */
    if (single)
        test_single(argv[0], source, build);
    else if (list != NULL) {
        shortlist = strrchr(list, '/');
        if (shortlist == NULL)
            shortlist = list;
        else
            shortlist++;
        printf(banner, shortlist);
        tests = read_test_list(list);
        status = test_batch(tests, source, build, verbose) ? 0 : 1;
    } else {
        tests = build_test_list(argv, argc);
        status = test_batch(tests, source, build, verbose) ? 0 : 1;
    }

    /* For valgrind cleanliness, free all our memory. */
    if (source_env != NULL) {
        putenv((char *) "C_TAP_SOURCE=");
        putenv((char *) "SOURCE=");
        free(c_tap_source_env);
        free(source_env);
    }
    if (build_env != NULL) {
        putenv((char *) "C_TAP_BUILD=");
        putenv((char *) "BUILD=");
        free(c_tap_build_env);
        free(build_env);
    }
    exit(status);
}