]>
jfr.im git - solanum.git/blob - tests/tap/basic.c
2 * Some utility routines for writing tests.
4 * Here are a variety of utility routines for writing tests compatible with
5 * the TAP protocol. All routines of the form ok() or is*() take a test
6 * number and some number of appropriate arguments, check to be sure the
7 * results match the expected output using the arguments, and print out
8 * something appropriate for that test number. Other utility routines help in
9 * constructing more complex tests, skipping tests, reporting errors, setting
10 * up the TAP output format, or finding things in the test environment.
12 * This file is part of C TAP Harness. The current version plus supporting
13 * documentation is at <https://www.eyrie.org/~eagle/software/c-tap-harness/>.
15 * Copyright 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016
16 * Russ Allbery <eagle@eyrie.org>
17 * Copyright 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2011, 2012, 2013, 2014
18 * The Board of Trustees of the Leland Stanford Junior University
20 * Permission is hereby granted, free of charge, to any person obtaining a
21 * copy of this software and associated documentation files (the "Software"),
22 * to deal in the Software without restriction, including without limitation
23 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 * and/or sell copies of the Software, and to permit persons to whom the
25 * Software is furnished to do so, subject to the following conditions:
27 * The above copyright notice and this permission notice shall be included in
28 * all copies or substantial portions of the Software.
30 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
31 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
33 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
36 * DEALINGS IN THE SOFTWARE.
48 # include <sys/stat.h>
50 #include <sys/types.h>
53 #include <tests/tap/basic.h>
55 /* Windows provides mkdir and rmdir under different names. */
57 # define mkdir(p, m) _mkdir(p)
58 # define rmdir(p) _rmdir(p)
62 * The test count. Always contains the number that will be used for the next
63 * test status. This is exported to callers of the library.
65 unsigned long testnum
= 1;
68 * Status information stored so that we can give a test summary at the end of
69 * the test case. We store the planned final test and the count of failures.
70 * We can get the highest test count from testnum.
72 static unsigned long _planned
= 0;
73 static unsigned long _failed
= 0;
76 * Store the PID of the process that called plan() and only summarize
77 * results when that process exits, so as to not misreport results in forked
80 static pid_t _process
= 0;
83 * If true, we're doing lazy planning and will print out the plan based on the
84 * last test number at the end of testing.
89 * If true, the test was aborted by calling bail(). Currently, this is only
90 * used to ensure that we pass a false value to any cleanup functions even if
91 * all tests to that point have passed.
93 static int _aborted
= 0;
96 * Registered cleanup functions. These are stored as a linked list and run in
97 * registered order by finish when the test program exits. Each function is
98 * passed a boolean value indicating whether all tests were successful.
100 struct cleanup_func
{
101 test_cleanup_func func
;
102 struct cleanup_func
*next
;
104 static struct cleanup_func
*cleanup_funcs
= NULL
;
107 * Registered diag files. Any output found in these files will be printed out
108 * as if it were passed to diag() before any other output we do. This allows
109 * background processes to log to a file and have that output interleaved with
117 struct diag_file
*next
;
119 static struct diag_file
*diag_files
= NULL
;
122 * Print a specified prefix and then the test description. Handles turning
123 * the argument list into a va_args structure suitable for passing to
124 * print_desc, which has to be done in a macro. Assumes that format is the
125 * argument immediately before the variadic arguments.
127 #define PRINT_DESC(prefix, format) \
129 if (format != NULL) { \
131 if (prefix != NULL) \
132 printf("%s", prefix); \
133 va_start(args, format); \
134 vprintf(format, args); \
141 * Form a new string by concatenating multiple strings. The arguments must be
142 * terminated by (const char *) 0.
144 * This function only exists because we can't assume asprintf. We can't
145 * simulate asprintf with snprintf because we're only assuming SUSv3, which
146 * does not require that snprintf with a NULL buffer return the required
147 * length. When those constraints are relaxed, this should be ripped out and
148 * replaced with asprintf or a more trivial replacement with snprintf.
151 concat(const char *first
, ...)
160 * Find the total memory required. Ensure we don't overflow length. See
161 * the comment for breallocarray for why we're using UINT_MAX here.
163 va_start(args
, first
);
164 for (string
= first
; string
!= NULL
; string
= va_arg(args
, const char *)) {
165 if (length
>= UINT_MAX
- strlen(string
))
166 bail("strings too long in concat");
167 length
+= strlen(string
);
172 /* Create the string. */
173 result
= bmalloc(length
);
174 va_start(args
, first
);
176 for (string
= first
; string
!= NULL
; string
= va_arg(args
, const char *)) {
177 memcpy(result
+ offset
, string
, strlen(string
));
178 offset
+= strlen(string
);
181 result
[offset
] = '\0';
187 * Check all registered diag_files for any output. We only print out the
188 * output if we see a complete line; otherwise, we wait for the next newline.
191 check_diag_files(void)
193 struct diag_file
*file
;
196 int size
, incomplete
;
199 * Walk through each file and read each line of output available. The
200 * general scheme here used is as follows: try to read a line of output at
201 * a time. If we get NULL, check for EOF; on EOF, advance to the next
204 * If we get some data, see if it ends in a newline. If it doesn't end in
205 * a newline, we have one of two cases: our buffer isn't large enough, in
206 * which case we resize it and try again, or we have incomplete data in
207 * the file, in which case we rewind the file and will try again next
210 for (file
= diag_files
; file
!= NULL
; file
= file
->next
) {
211 clearerr(file
->file
);
213 /* Store the current position in case we have to rewind. */
214 if (fgetpos(file
->file
, &where
) < 0)
215 sysbail("cannot get position in %s", file
->name
);
217 /* Continue until we get EOF or an incomplete line of data. */
219 while (!feof(file
->file
) && !incomplete
) {
220 size
= file
->bufsize
> INT_MAX
? INT_MAX
: (int) file
->bufsize
;
221 if (fgets(file
->buffer
, size
, file
->file
) == NULL
) {
222 if (ferror(file
->file
))
223 sysbail("cannot read from %s", file
->name
);
228 * See if the line ends in a newline. If not, see which error
229 * case we have. Use UINT_MAX as a substitute for SIZE_MAX (see
230 * the comment for breallocarray).
232 length
= strlen(file
->buffer
);
233 if (file
->buffer
[length
- 1] != '\n') {
234 if (length
< file
->bufsize
- 1)
237 if (file
->bufsize
>= UINT_MAX
- BUFSIZ
)
238 sysbail("line too long in %s", file
->name
);
239 file
->bufsize
+= BUFSIZ
;
240 file
->buffer
= brealloc(file
->buffer
, file
->bufsize
);
244 * On either incomplete lines or too small of a buffer, rewind
245 * and read the file again (on the next pass, if incomplete).
246 * It's simpler than trying to double-buffer the file.
248 if (fsetpos(file
->file
, &where
) < 0)
249 sysbail("cannot set position in %s", file
->name
);
253 /* We saw a complete line. Print it out. */
254 printf("# %s", file
->buffer
);
261 * Our exit handler. Called on completion of the test to report a summary of
262 * results provided we're still in the original process. This also handles
263 * printing out the plan if we used plan_lazy(), although that's suppressed if
264 * we never ran a test (due to an early bail, for example), and running any
265 * registered cleanup functions.
270 int success
, primary
;
271 struct cleanup_func
*current
;
272 unsigned long highest
= testnum
- 1;
273 struct diag_file
*file
, *tmp
;
275 /* Check for pending diag_file output. */
278 /* Free the diag_files. */
280 while (file
!= NULL
) {
291 * Determine whether all tests were successful, which is needed before
292 * calling cleanup functions since we pass that fact to the functions.
294 if (_planned
== 0 && _lazy
)
296 success
= (!_aborted
&& _planned
== highest
&& _failed
== 0);
299 * If there are any registered cleanup functions, we run those first. We
300 * always run them, even if we didn't run a test. Don't do anything
301 * except free the diag_files and call cleanup functions if we aren't the
302 * primary process (the process in which plan or plan_lazy was called),
303 * and tell the cleanup functions that fact.
305 primary
= (_process
== 0 || getpid() == _process
);
306 while (cleanup_funcs
!= NULL
) {
307 cleanup_funcs
->func(success
, primary
);
308 current
= cleanup_funcs
;
309 cleanup_funcs
= cleanup_funcs
->next
;
315 /* Don't do anything further if we never planned a test. */
319 /* If we're aborting due to bail, don't print summaries. */
323 /* Print out the lazy plan if needed. */
325 if (_lazy
&& _planned
> 0)
326 printf("1..%lu\n", _planned
);
328 /* Print out a summary of the results. */
329 if (_planned
> highest
)
330 diag("Looks like you planned %lu test%s but only ran %lu", _planned
,
331 (_planned
> 1 ? "s" : ""), highest
);
332 else if (_planned
< highest
)
333 diag("Looks like you planned %lu test%s but ran %lu extra", _planned
,
334 (_planned
> 1 ? "s" : ""), highest
- _planned
);
335 else if (_failed
> 0)
336 diag("Looks like you failed %lu test%s of %lu", _failed
,
337 (_failed
> 1 ? "s" : ""), _planned
);
338 else if (_planned
!= 1)
339 diag("All %lu tests successful or skipped", _planned
);
341 diag("%lu test successful or skipped", _planned
);
346 * Initialize things. Turns on line buffering on stdout and then prints out
347 * the number of tests in the test suite. We intentionally don't check for
348 * pending diag_file output here, since it should really come after the plan.
351 plan(unsigned long count
)
353 if (setvbuf(stdout
, NULL
, _IOLBF
, BUFSIZ
) != 0)
354 sysdiag("cannot set stdout to line buffered");
356 printf("1..%lu\n", count
);
360 if (atexit(finish
) != 0) {
361 sysdiag("cannot register exit handler");
362 diag("cleanups will not be run");
368 * Initialize things for lazy planning, where we'll automatically print out a
369 * plan at the end of the program. Turns on line buffering on stdout as well.
374 if (setvbuf(stdout
, NULL
, _IOLBF
, BUFSIZ
) != 0)
375 sysdiag("cannot set stdout to line buffered");
379 if (atexit(finish
) != 0)
380 sysbail("cannot register exit handler to display plan");
385 * Skip the entire test suite and exits. Should be called instead of plan(),
386 * not after it, since it prints out a special plan line. Ignore diag_file
387 * output here, since it's not clear if it's allowed before the plan.
390 skip_all(const char *format
, ...)
393 printf("1..0 # skip");
394 PRINT_DESC(" ", format
);
401 * Takes a boolean success value and assumes the test passes if that value
402 * is true and fails if that value is false.
405 ok(int success
, const char *format
, ...)
409 printf("%sok %lu", success
? "" : "not ", testnum
++);
412 PRINT_DESC(" - ", format
);
419 * Same as ok(), but takes the format arguments as a va_list.
422 okv(int success
, const char *format
, va_list args
)
426 printf("%sok %lu", success
? "" : "not ", testnum
++);
429 if (format
!= NULL
) {
431 vprintf(format
, args
);
442 skip(const char *reason
, ...)
446 printf("ok %lu # skip", testnum
++);
447 PRINT_DESC(" ", reason
);
453 * Report the same status on the next count tests.
456 ok_block(unsigned long count
, int success
, const char *format
, ...)
462 for (i
= 0; i
< count
; i
++) {
463 printf("%sok %lu", success
? "" : "not ", testnum
++);
466 PRINT_DESC(" - ", format
);
474 * Skip the next count tests.
477 skip_block(unsigned long count
, const char *reason
, ...)
483 for (i
= 0; i
< count
; i
++) {
484 printf("ok %lu # skip", testnum
++);
485 PRINT_DESC(" ", reason
);
492 * Takes an expected boolean value and a seen boolean value and assumes the
493 * test passes if the truth value of both match.
496 is_bool(int wanted
, int seen
, const char *format
, ...)
502 success
= (!!wanted
== !!seen
);
504 printf("ok %lu", testnum
++);
506 diag("wanted: %s", !!wanted
? "true" : "false");
507 diag(" seen: %s", !!seen
? "true" : "false");
508 printf("not ok %lu", testnum
++);
511 PRINT_DESC(" - ", format
);
518 * Takes an expected integer and a seen integer and assumes the test passes
519 * if those two numbers match.
522 is_int(long wanted
, long seen
, const char *format
, ...)
528 success
= (wanted
== seen
);
530 printf("ok %lu", testnum
++);
532 diag("wanted: %ld", wanted
);
533 diag(" seen: %ld", seen
);
534 printf("not ok %lu", testnum
++);
537 PRINT_DESC(" - ", format
);
544 * Takes a string and what the string should be, and assumes the test passes
545 * if those strings match (using strcmp).
548 is_string(const char *wanted
, const char *seen
, const char *format
, ...)
558 success
= (strcmp(wanted
, seen
) == 0);
560 printf("ok %lu", testnum
++);
562 diag("wanted: %s", wanted
);
563 diag(" seen: %s", seen
);
564 printf("not ok %lu", testnum
++);
567 PRINT_DESC(" - ", format
);
574 * Takes an expected unsigned long and a seen unsigned long and assumes the
575 * test passes if the two numbers match. Otherwise, reports them in hex.
578 is_hex(unsigned long wanted
, unsigned long seen
, const char *format
, ...)
584 success
= (wanted
== seen
);
586 printf("ok %lu", testnum
++);
588 diag("wanted: %lx", (unsigned long) wanted
);
589 diag(" seen: %lx", (unsigned long) seen
);
590 printf("not ok %lu", testnum
++);
593 PRINT_DESC(" - ", format
);
600 * Bail out with an error.
603 bail(const char *format
, ...)
611 printf("Bail out! ");
612 va_start(args
, format
);
613 vprintf(format
, args
);
621 * Bail out with an error, appending strerror(errno).
624 sysbail(const char *format
, ...)
633 printf("Bail out! ");
634 va_start(args
, format
);
635 vprintf(format
, args
);
637 printf(": %s\n", strerror(oerrno
));
643 * Report a diagnostic to stderr. Always returns 1 to allow embedding in
644 * compound statements.
647 diag(const char *format
, ...)
655 va_start(args
, format
);
656 vprintf(format
, args
);
664 * Report a diagnostic to stderr, appending strerror(errno). Always returns 1
665 * to allow embedding in compound statements.
668 sysdiag(const char *format
, ...)
677 va_start(args
, format
);
678 vprintf(format
, args
);
680 printf(": %s\n", strerror(oerrno
));
686 * Register a new file for diag_file processing.
689 diag_file_add(const char *name
)
691 struct diag_file
*file
, *prev
;
693 file
= bcalloc(1, sizeof(struct diag_file
));
694 file
->name
= bstrdup(name
);
695 file
->file
= fopen(file
->name
, "r");
696 if (file
->file
== NULL
)
697 sysbail("cannot open %s", name
);
698 file
->buffer
= bmalloc(BUFSIZ
);
699 file
->bufsize
= BUFSIZ
;
700 if (diag_files
== NULL
)
703 for (prev
= diag_files
; prev
->next
!= NULL
; prev
= prev
->next
)
711 * Remove a file from diag_file processing. If the file is not found, do
712 * nothing, since there are some situations where it can be removed twice
713 * (such as if it's removed from a cleanup function, since cleanup functions
714 * are called after freeing all the diag_files).
717 diag_file_remove(const char *name
)
719 struct diag_file
*file
;
720 struct diag_file
**prev
= &diag_files
;
722 for (file
= diag_files
; file
!= NULL
; file
= file
->next
) {
723 if (strcmp(file
->name
, name
) == 0) {
737 * Allocate cleared memory, reporting a fatal error with bail on failure.
740 bcalloc(size_t n
, size_t size
)
746 sysbail("failed to calloc %lu", (unsigned long)(n
* size
));
752 * Allocate memory, reporting a fatal error with bail on failure.
761 sysbail("failed to malloc %lu", (unsigned long) size
);
767 * Reallocate memory, reporting a fatal error with bail on failure.
770 brealloc(void *p
, size_t size
)
772 p
= realloc(p
, size
);
774 sysbail("failed to realloc %lu bytes", (unsigned long) size
);
780 * The same as brealloc, but determine the size by multiplying an element
781 * count by a size, similar to calloc. The multiplication is checked for
784 * We should technically use SIZE_MAX here for the overflow check, but
785 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
786 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we
787 * can assume that UINT_MAX <= SIZE_MAX.
789 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
790 * I disbelieve in the existence of such systems and they will have to cope
791 * without overflow checks.)
794 breallocarray(void *p
, size_t n
, size_t size
)
796 if (n
> 0 && UINT_MAX
/ n
<= size
)
797 bail("reallocarray too large");
798 p
= realloc(p
, n
* size
);
800 sysbail("failed to realloc %lu bytes", (unsigned long) (n
* size
));
806 * Copy a string, reporting a fatal error with bail on failure.
809 bstrdup(const char *s
)
817 sysbail("failed to strdup %lu bytes", (unsigned long) len
);
824 * Copy up to n characters of a string, reporting a fatal error with bail on
825 * failure. Don't use the system strndup function, since it may not exist and
826 * the TAP library doesn't assume any portability support.
829 bstrndup(const char *s
, size_t n
)
835 /* Don't assume that the source string is nul-terminated. */
836 for (p
= s
; (size_t) (p
- s
) < n
&& *p
!= '\0'; p
++)
838 length
= (size_t) (p
- s
);
839 copy
= malloc(length
+ 1);
841 sysbail("failed to strndup %lu bytes", (unsigned long) length
);
842 memcpy(copy
, s
, length
);
849 * Locate a test file. Given the partial path to a file, look under
850 * C_TAP_BUILD and then C_TAP_SOURCE for the file and return the full path to
851 * the file. Returns NULL if the file doesn't exist. A non-NULL return
852 * should be freed with test_file_path_free().
855 test_file_path(const char *file
)
859 const char *envs
[] = { "C_TAP_BUILD", "C_TAP_SOURCE", NULL
};
862 for (i
= 0; envs
[i
] != NULL
; i
++) {
863 base
= getenv(envs
[i
]);
866 path
= concat(base
, "/", file
, (const char *) 0);
867 if (access(path
, R_OK
) == 0)
877 * Free a path returned from test_file_path(). This function exists primarily
878 * for Windows, where memory must be freed from the same library domain that
879 * it was allocated from.
882 test_file_path_free(char *path
)
889 * Create a temporary directory, tmp, under C_TAP_BUILD if set and the current
890 * directory if it does not. Returns the path to the temporary directory in
891 * newly allocated memory, and calls bail on any failure. The return value
892 * should be freed with test_tmpdir_free.
894 * This function uses sprintf because it attempts to be independent of all
895 * other portability layers. The use immediately after a memory allocation
896 * should be safe without using snprintf or strlcpy/strlcat.
904 build
= getenv("C_TAP_BUILD");
907 path
= concat(build
, "/tmp", (const char *) 0);
908 if (access(path
, X_OK
) < 0)
909 if (mkdir(path
, 0777) < 0)
910 sysbail("error creating temporary directory %s", path
);
916 * Free a path returned from test_tmpdir() and attempt to remove the
917 * directory. If we can't delete the directory, don't worry; something else
918 * that hasn't yet cleaned up may still be using it.
921 test_tmpdir_free(char *path
)
930 * Register a cleanup function that is called when testing ends. All such
931 * registered functions will be run by finish.
934 test_cleanup_register(test_cleanup_func func
)
936 struct cleanup_func
*cleanup
, **last
;
938 cleanup
= bmalloc(sizeof(struct cleanup_func
));
939 cleanup
->func
= func
;
940 cleanup
->next
= NULL
;
941 last
= &cleanup_funcs
;
942 while (*last
!= NULL
)
943 last
= &(*last
)->next
;