]>
jfr.im git - solanum.git/blob - tests/tap/basic.c
2 * Some utility routines for writing tests.
4 * Here are a variety of utility routines for writing tests compatible with
5 * the TAP protocol. All routines of the form ok() or is*() take a test
6 * number and some number of appropriate arguments, check to be sure the
7 * results match the expected output using the arguments, and print out
8 * something appropriate for that test number. Other utility routines help in
9 * constructing more complex tests, skipping tests, reporting errors, setting
10 * up the TAP output format, or finding things in the test environment.
12 * This file is part of C TAP Harness. The current version plus supporting
13 * documentation is at <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
15 * Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016
16 * Russ Allbery <eagle@eyrie.org>
17 * Copyright 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2011, 2012, 2013, 2014
18 * The Board of Trustees of the Leland Stanford Junior University
20 * Permission is hereby granted, free of charge, to any person obtaining a
21 * copy of this software and associated documentation files (the "Software"),
22 * to deal in the Software without restriction, including without limitation
23 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 * and/or sell copies of the Software, and to permit persons to whom the
25 * Software is furnished to do so, subject to the following conditions:
27 * The above copyright notice and this permission notice shall be included in
28 * all copies or substantial portions of the Software.
30 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
31 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
33 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
36 * DEALINGS IN THE SOFTWARE.
46 #include <sys/types.h>
48 #include <tests/tap/basic.h>
51 * The test count. Always contains the number that will be used for the next
52 * test status. This is exported to callers of the library.
54 unsigned long testnum
= 1;
57 * Status information stored so that we can give a test summary at the end of
58 * the test case. We store the planned final test and the count of failures.
59 * We can get the highest test count from testnum.
61 static unsigned long _planned
= 0;
62 static unsigned long _failed
= 0;
65 * Store the PID of the process that called plan() and only summarize
66 * results when that process exits, so as to not misreport results in forked
69 static pid_t _process
= 0;
72 * If true, we're doing lazy planning and will print out the plan based on the
73 * last test number at the end of testing.
78 * If true, the test was aborted by calling bail(). Currently, this is only
79 * used to ensure that we pass a false value to any cleanup functions even if
80 * all tests to that point have passed.
82 static int _aborted
= 0;
85 * Registered cleanup functions. These are stored as a linked list and run in
86 * registered order by finish when the test program exits. Each function is
87 * passed a boolean value indicating whether all tests were successful.
90 test_cleanup_func func
;
91 struct cleanup_func
*next
;
93 static struct cleanup_func
*cleanup_funcs
= NULL
;
96 * Registered diag files. Any output found in these files will be printed out
97 * as if it were passed to diag() before any other output we do. This allows
98 * background processes to log to a file and have that output interleaved with
106 struct diag_file
*next
;
108 static struct diag_file
*diag_files
= NULL
;
111 * Print a specified prefix and then the test description. Handles turning
112 * the argument list into a va_args structure suitable for passing to
113 * print_desc, which has to be done in a macro. Assumes that format is the
114 * argument immediately before the variadic arguments.
116 #define PRINT_DESC(prefix, format) \
118 if (format != NULL) { \
120 if (prefix != NULL) \
121 printf("%s", prefix); \
122 va_start(args, format); \
123 vprintf(format, args); \
130 * Form a new string by concatenating multiple strings. The arguments must be
131 * terminated by (const char *) 0.
133 * This function only exists because we can't assume asprintf. We can't
134 * simulate asprintf with snprintf because we're only assuming SUSv3, which
135 * does not require that snprintf with a NULL buffer return the required
136 * length. When those constraints are relaxed, this should be ripped out and
137 * replaced with asprintf or a more trivial replacement with snprintf.
140 concat(const char *first
, ...)
149 * Find the total memory required. Ensure we don't overflow length. See
150 * the comment for breallocarray for why we're using UINT_MAX here.
152 va_start(args
, first
);
153 for (string
= first
; string
!= NULL
; string
= va_arg(args
, const char *)) {
154 if (length
>= UINT_MAX
- strlen(string
))
155 bail("strings too long in concat");
156 length
+= strlen(string
);
161 /* Create the string. */
162 result
= bmalloc(length
);
163 va_start(args
, first
);
165 for (string
= first
; string
!= NULL
; string
= va_arg(args
, const char *)) {
166 memcpy(result
+ offset
, string
, strlen(string
));
167 offset
+= strlen(string
);
170 result
[offset
] = '\0';
176 * Check all registered diag_files for any output. We only print out the
177 * output if we see a complete line; otherwise, we wait for the next newline.
180 check_diag_files(void)
182 struct diag_file
*file
;
185 int size
, incomplete
;
188 * Walk through each file and read each line of output available. The
189 * general scheme here used is as follows: try to read a line of output at
190 * a time. If we get NULL, check for EOF; on EOF, advance to the next
193 * If we get some data, see if it ends in a newline. If it doesn't end in
194 * a newline, we have one of two cases: our buffer isn't large enough, in
195 * which case we resize it and try again, or we have incomplete data in
196 * the file, in which case we rewind the file and will try again next
199 for (file
= diag_files
; file
!= NULL
; file
= file
->next
) {
200 clearerr(file
->file
);
202 /* Store the current position in case we have to rewind. */
203 if (fgetpos(file
->file
, &where
) < 0)
204 sysbail("cannot get position in %s", file
->name
);
206 /* Continue until we get EOF or an incomplete line of data. */
208 while (!feof(file
->file
) && !incomplete
) {
209 size
= file
->bufsize
> INT_MAX
? INT_MAX
: (int) file
->bufsize
;
210 if (fgets(file
->buffer
, size
, file
->file
) == NULL
) {
211 if (ferror(file
->file
))
212 sysbail("cannot read from %s", file
->name
);
217 * See if the line ends in a newline. If not, see which error
218 * case we have. Use UINT_MAX as a substitute for SIZE_MAX (see
219 * the comment for breallocarray).
221 length
= strlen(file
->buffer
);
222 if (file
->buffer
[length
- 1] != '\n') {
223 if (length
< file
->bufsize
- 1)
226 if (file
->bufsize
>= UINT_MAX
- BUFSIZ
)
227 sysbail("line too long in %s", file
->name
);
228 file
->bufsize
+= BUFSIZ
;
229 file
->buffer
= brealloc(file
->buffer
, file
->bufsize
);
233 * On either incomplete lines or too small of a buffer, rewind
234 * and read the file again (on the next pass, if incomplete).
235 * It's simpler than trying to double-buffer the file.
237 if (fsetpos(file
->file
, &where
) < 0)
238 sysbail("cannot set position in %s", file
->name
);
242 /* We saw a complete line. Print it out. */
243 printf("# %s", file
->buffer
);
250 * Our exit handler. Called on completion of the test to report a summary of
251 * results provided we're still in the original process. This also handles
252 * printing out the plan if we used plan_lazy(), although that's suppressed if
253 * we never ran a test (due to an early bail, for example), and running any
254 * registered cleanup functions.
259 int success
, primary
;
260 struct cleanup_func
*current
;
261 unsigned long highest
= testnum
- 1;
262 struct diag_file
*file
, *tmp
;
264 /* Check for pending diag_file output. */
267 /* Free the diag_files. */
269 while (file
!= NULL
) {
280 * Determine whether all tests were successful, which is needed before
281 * calling cleanup functions since we pass that fact to the functions.
283 if (_planned
== 0 && _lazy
)
285 success
= (!_aborted
&& _planned
== highest
&& _failed
== 0);
288 * If there are any registered cleanup functions, we run those first. We
289 * always run them, even if we didn't run a test. Don't do anything
290 * except free the diag_files and call cleanup functions if we aren't the
291 * primary process (the process in which plan or plan_lazy was called),
292 * and tell the cleanup functions that fact.
294 primary
= (_process
== 0 || getpid() == _process
);
295 while (cleanup_funcs
!= NULL
) {
296 cleanup_funcs
->func(success
, primary
);
297 current
= cleanup_funcs
;
298 cleanup_funcs
= cleanup_funcs
->next
;
304 /* Don't do anything further if we never planned a test. */
308 /* If we're aborting due to bail, don't print summaries. */
312 /* Print out the lazy plan if needed. */
314 if (_lazy
&& _planned
> 0)
315 printf("1..%lu\n", _planned
);
317 /* Print out a summary of the results. */
318 if (_planned
> highest
)
319 diag("Looks like you planned %lu test%s but only ran %lu", _planned
,
320 (_planned
> 1 ? "s" : ""), highest
);
321 else if (_planned
< highest
)
322 diag("Looks like you planned %lu test%s but ran %lu extra", _planned
,
323 (_planned
> 1 ? "s" : ""), highest
- _planned
);
324 else if (_failed
> 0)
325 diag("Looks like you failed %lu test%s of %lu", _failed
,
326 (_failed
> 1 ? "s" : ""), _planned
);
327 else if (_planned
!= 1)
328 diag("All %lu tests successful or skipped", _planned
);
330 diag("%lu test successful or skipped", _planned
);
335 * Initialize things. Turns on line buffering on stdout and then prints out
336 * the number of tests in the test suite. We intentionally don't check for
337 * pending diag_file output here, since it should really come after the plan.
340 plan(unsigned long count
)
342 if (setvbuf(stdout
, NULL
, _IOLBF
, BUFSIZ
) != 0)
343 sysdiag("cannot set stdout to line buffered");
345 printf("1..%lu\n", count
);
349 if (atexit(finish
) != 0) {
350 sysdiag("cannot register exit handler");
351 diag("cleanups will not be run");
357 * Initialize things for lazy planning, where we'll automatically print out a
358 * plan at the end of the program. Turns on line buffering on stdout as well.
363 if (setvbuf(stdout
, NULL
, _IOLBF
, BUFSIZ
) != 0)
364 sysdiag("cannot set stdout to line buffered");
368 if (atexit(finish
) != 0)
369 sysbail("cannot register exit handler to display plan");
374 * Skip the entire test suite and exits. Should be called instead of plan(),
375 * not after it, since it prints out a special plan line. Ignore diag_file
376 * output here, since it's not clear if it's allowed before the plan.
379 skip_all(const char *format
, ...)
382 printf("1..0 # skip");
383 PRINT_DESC(" ", format
);
390 * Takes a boolean success value and assumes the test passes if that value
391 * is true and fails if that value is false.
394 ok(int success
, const char *format
, ...)
398 printf("%sok %lu", success
? "" : "not ", testnum
++);
401 PRINT_DESC(" - ", format
);
408 * Same as ok(), but takes the format arguments as a va_list.
411 okv(int success
, const char *format
, va_list args
)
415 printf("%sok %lu", success
? "" : "not ", testnum
++);
418 if (format
!= NULL
) {
420 vprintf(format
, args
);
431 skip(const char *reason
, ...)
435 printf("ok %lu # skip", testnum
++);
436 PRINT_DESC(" ", reason
);
442 * Report the same status on the next count tests.
445 ok_block(unsigned long count
, int success
, const char *format
, ...)
451 for (i
= 0; i
< count
; i
++) {
452 printf("%sok %lu", success
? "" : "not ", testnum
++);
455 PRINT_DESC(" - ", format
);
463 * Skip the next count tests.
466 skip_block(unsigned long count
, const char *reason
, ...)
472 for (i
= 0; i
< count
; i
++) {
473 printf("ok %lu # skip", testnum
++);
474 PRINT_DESC(" ", reason
);
481 * Takes an expected boolean value and a seen boolean value and assumes the
482 * test passes if the truth value of both match.
485 is_bool(int wanted
, int seen
, const char *format
, ...)
491 success
= (!!wanted
== !!seen
);
493 printf("ok %lu", testnum
++);
495 diag("wanted: %s", !!wanted
? "true" : "false");
496 diag(" seen: %s", !!seen
? "true" : "false");
497 printf("not ok %lu", testnum
++);
500 PRINT_DESC(" - ", format
);
507 * Takes an expected integer and a seen integer and assumes the test passes
508 * if those two numbers match.
511 is_int(long wanted
, long seen
, const char *format
, ...)
517 success
= (wanted
== seen
);
519 printf("ok %lu", testnum
++);
521 diag("wanted: %ld", wanted
);
522 diag(" seen: %ld", seen
);
523 printf("not ok %lu", testnum
++);
526 PRINT_DESC(" - ", format
);
533 * Takes a string and what the string should be, and assumes the test passes
534 * if those strings match (using strcmp).
537 is_string(const char *wanted
, const char *seen
, const char *format
, ...)
547 success
= (strcmp(wanted
, seen
) == 0);
549 printf("ok %lu", testnum
++);
551 diag("wanted: %s", wanted
);
552 diag(" seen: %s", seen
);
553 printf("not ok %lu", testnum
++);
556 PRINT_DESC(" - ", format
);
563 * Takes an expected unsigned long and a seen unsigned long and assumes the
564 * test passes if the two numbers match. Otherwise, reports them in hex.
567 is_hex(unsigned long wanted
, unsigned long seen
, const char *format
, ...)
573 success
= (wanted
== seen
);
575 printf("ok %lu", testnum
++);
577 diag("wanted: %lx", (unsigned long) wanted
);
578 diag(" seen: %lx", (unsigned long) seen
);
579 printf("not ok %lu", testnum
++);
582 PRINT_DESC(" - ", format
);
589 * Bail out with an error.
592 bail(const char *format
, ...)
600 printf("Bail out! ");
601 va_start(args
, format
);
602 vprintf(format
, args
);
610 * Bail out with an error, appending strerror(errno).
613 sysbail(const char *format
, ...)
622 printf("Bail out! ");
623 va_start(args
, format
);
624 vprintf(format
, args
);
626 printf(": %s\n", strerror(oerrno
));
632 * Report a diagnostic to stderr. Always returns 1 to allow embedding in
633 * compound statements.
636 diag(const char *format
, ...)
644 va_start(args
, format
);
645 vprintf(format
, args
);
653 * Report a diagnostic to stderr, appending strerror(errno). Always returns 1
654 * to allow embedding in compound statements.
657 sysdiag(const char *format
, ...)
666 va_start(args
, format
);
667 vprintf(format
, args
);
669 printf(": %s\n", strerror(oerrno
));
675 * Register a new file for diag_file processing.
678 diag_file_add(const char *name
)
680 struct diag_file
*file
, *prev
;
682 file
= bcalloc(1, sizeof(struct diag_file
));
683 file
->name
= bstrdup(name
);
684 file
->file
= fopen(file
->name
, "r");
685 if (file
->file
== NULL
)
686 sysbail("cannot open %s", name
);
687 file
->buffer
= bmalloc(BUFSIZ
);
688 file
->bufsize
= BUFSIZ
;
689 if (diag_files
== NULL
)
692 for (prev
= diag_files
; prev
->next
!= NULL
; prev
= prev
->next
)
700 * Remove a file from diag_file processing. If the file is not found, do
701 * nothing, since there are some situations where it can be removed twice
702 * (such as if it's removed from a cleanup function, since cleanup functions
703 * are called after freeing all the diag_files).
706 diag_file_remove(const char *name
)
708 struct diag_file
*file
;
709 struct diag_file
**prev
= &diag_files
;
711 for (file
= diag_files
; file
!= NULL
; file
= file
->next
) {
712 if (strcmp(file
->name
, name
) == 0) {
726 * Allocate cleared memory, reporting a fatal error with bail on failure.
729 bcalloc(size_t n
, size_t size
)
735 sysbail("failed to calloc %lu", (unsigned long)(n
* size
));
741 * Allocate memory, reporting a fatal error with bail on failure.
750 sysbail("failed to malloc %lu", (unsigned long) size
);
756 * Reallocate memory, reporting a fatal error with bail on failure.
759 brealloc(void *p
, size_t size
)
761 p
= realloc(p
, size
);
763 sysbail("failed to realloc %lu bytes", (unsigned long) size
);
769 * The same as brealloc, but determine the size by multiplying an element
770 * count by a size, similar to calloc. The multiplication is checked for
773 * We should technically use SIZE_MAX here for the overflow check, but
774 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
775 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we
776 * can assume that UINT_MAX <= SIZE_MAX.
778 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
779 * I disbelieve in the existence of such systems and they will have to cope
780 * without overflow checks.)
783 breallocarray(void *p
, size_t n
, size_t size
)
785 if (n
> 0 && UINT_MAX
/ n
<= size
)
786 bail("reallocarray too large");
787 p
= realloc(p
, n
* size
);
789 sysbail("failed to realloc %lu bytes", (unsigned long) (n
* size
));
795 * Copy a string, reporting a fatal error with bail on failure.
798 bstrdup(const char *s
)
806 sysbail("failed to strdup %lu bytes", (unsigned long) len
);
813 * Copy up to n characters of a string, reporting a fatal error with bail on
814 * failure. Don't use the system strndup function, since it may not exist and
815 * the TAP library doesn't assume any portability support.
818 bstrndup(const char *s
, size_t n
)
824 /* Don't assume that the source string is nul-terminated. */
825 for (p
= s
; (size_t) (p
- s
) < n
&& *p
!= '\0'; p
++)
827 length
= (size_t) (p
- s
);
828 copy
= malloc(length
+ 1);
830 sysbail("failed to strndup %lu bytes", (unsigned long) length
);
831 memcpy(copy
, s
, length
);
838 * Locate a test file. Given the partial path to a file, look under
839 * C_TAP_BUILD and then C_TAP_SOURCE for the file and return the full path to
840 * the file. Returns NULL if the file doesn't exist. A non-NULL return
841 * should be freed with free().
844 test_file_path(const char *file
)
848 const char *envs
[] = { "C_TAP_BUILD", "C_TAP_SOURCE", NULL
};
851 for (i
= 0; envs
[i
] != NULL
; i
++) {
852 base
= getenv(envs
[i
]);
855 path
= concat(base
, "/", file
, (const char *) 0);
856 if (access(path
, R_OK
) == 0)
866 * Create a temporary directory, tmp, under C_TAP_BUILD if set and the current
867 * directory if it does not. Returns the path to the temporary directory in
868 * newly allocated memory, and calls bail on any failure. The return value
869 * should be freed with test_tmpdir_free.
871 * This function uses sprintf because it attempts to be independent of all
872 * other portability layers. The use immediately after a memory allocation
873 * should be safe without using snprintf or strlcpy/strlcat.
881 build
= getenv("C_TAP_BUILD");
884 path
= concat(build
, "/tmp", (const char *) 0);
885 if (access(path
, X_OK
) < 0)
886 if (mkdir(path
, 0777) < 0)
887 sysbail("error creating temporary directory %s", path
);
893 * Free a path returned from test_tmpdir() and attempt to remove the
894 * directory. If we can't delete the directory, don't worry; something else
895 * that hasn't yet cleaned up may still be using it.
898 test_tmpdir_free(char *path
)
907 * Register a cleanup function that is called when testing ends. All such
908 * registered functions will be run by finish.
911 test_cleanup_register(test_cleanup_func func
)
913 struct cleanup_func
*cleanup
, **last
;
915 cleanup
= bmalloc(sizeof(struct cleanup_func
));
916 cleanup
->func
= func
;
917 cleanup
->next
= NULL
;
918 last
= &cleanup_funcs
;
919 while (*last
!= NULL
)
920 last
= &(*last
)->next
;