RFC [1/2] test-in-container infrastructure

Message ID xno9kzm6fk.fsf@greed.delorie.com
State New
Headers show
Series
  • RFC [1/2] test-in-container infrastructure
Related show

Commit Message

DJ Delorie Feb. 8, 2018, 11:02 p.m.

Patch

diff --git a/Makefile b/Makefile
index bea4e27f8d..4f0fa17f8b 100644
--- a/Makefile
+++ b/Makefile
@@ -297,6 +297,44 @@  define summarize-tests
 @! egrep -q -v '^(X?PASS|XFAIL|UNSUPPORTED):' $(objpfx)$1
 endef
 
+# The intention here is to do ONE install of our build into the
+# testroot.pristine/ directory, then rsync (internal to
+# support/test-container) that to testroot.root/ at the start of each
+# test.  That way we can promise each test a "clean" install, without
+# having to do the install for each test.
+#
+# In addition, we have to copy some files into this root in addition
+# to what glibc installs.  For example, many tests require /bin/sh be
+# present, and any shared objects that /bin/sh depends on.  We also
+# build a "test" program in either C or (if available) C++ just so we
+# can copy in any shared objects that GCC-compiled programs depend on.
+
+$(tests-container) $(addsuffix /tests,$(subdirs)) : $(objpfx)testroot.pristine/ready.ts
+$(objpfx)testroot.pristine/ready.ts :
+	test -d $(objpfx)testroot.pristine || \
+	  mkdir $(objpfx)testroot.pristine
+	# We need a working /bin/sh for some of the tests.
+	test -d $(objpfx)testroot.pristine/bin || \
+	  mkdir $(objpfx)testroot.pristine/bin
+	cp /bin/sh $(objpfx)testroot.pristine/bin/sh
+	# Copy these DSOs first so we can overwrite them with our own.
+	set -vx ; for dso in `ldd /bin/sh \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	set -vx ; for dso in `ldd $(objpfx)support/links-dso-program \
+	                      | grep / | sed 's/^[^/]*//' | sed 's/ .*//'` ;\
+	  do \
+	    test -d `dirname $(objpfx)testroot.pristine$$dso` || \
+	      mkdir -p `dirname $(objpfx)testroot.pristine$$dso` ;\
+	    cp $$dso $(objpfx)testroot.pristine$$dso ;\
+	  done
+	$(MAKE) install DESTDIR=$(objpfx)testroot.pristine
+	touch $(objpfx)testroot.pristine/ready.ts
+
 tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
diff --git a/Makerules b/Makerules
index ef6abeac6d..893045cf0c 100644
--- a/Makerules
+++ b/Makerules
@@ -1365,7 +1365,7 @@  xcheck: xtests
 # The only difference between MODULE_NAME=testsuite and MODULE_NAME=nonlib is
 # that almost all internal declarations from config.h, libc-symbols.h, and
 # include/*.h are not available to 'testsuite' code, but are to 'nonlib' code.
-all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras))
+all-testsuite := $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(tests-container))
 ifneq (,$(all-testsuite))
 cpp-srcs-left = $(all-testsuite)
 lib := testsuite
diff --git a/Rules b/Rules
index 706c8a749d..8b30b23479 100644
--- a/Rules
+++ b/Rules
@@ -130,12 +130,12 @@  others: $(py-const)
 
 ifeq ($(run-built-tests),no)
 tests: $(addprefix $(objpfx),$(filter-out $(tests-unsupported), \
-                                          $(tests) $(tests-internal)) \
+                                          $(tests) $(tests-internal) $(tests-container)) \
 			     $(test-srcs)) $(tests-special) \
 			     $(tests-printers-programs)
 xtests: tests $(xtests-special)
 else
-tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) \
+tests: $(tests:%=$(objpfx)%.out) $(tests-internal:%=$(objpfx)%.out) $(tests-container:%=$(objpfx)%.out) \
        $(tests-special) $(tests-printers-out)
 xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special)
 endif
@@ -149,7 +149,7 @@  tests-expected = $(tests) $(tests-internal) $(tests-printers)
 endif
 tests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
-	  $(sort $(tests-expected) $(tests-special-notdir:.out=)) \
+	  $(sort $(tests-expected) $(tests-special-notdir:.out=) $(tests-container)) \
 	  > $(objpfx)subdir-tests.sum
 xtests:
 	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
@@ -158,7 +158,7 @@  xtests:
 
 ifeq ($(build-programs),yes)
 binaries-all-notests = $(others) $(sysdep-others)
-binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs)
+binaries-all-tests = $(tests) $(tests-internal) $(xtests) $(test-srcs) $(tests-container)
 binaries-all = $(binaries-all-notests) $(binaries-all-tests)
 binaries-static-notests = $(others-static)
 binaries-static-tests = $(tests-static) $(xtests-static)
@@ -248,6 +248,14 @@  $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
 	$(make-test-out) > $@; \
 	$(evaluate-test)
 
+
+# Any tests that require an isolated container (chroot) in which to
+# run, should be added to tests-container.
+$(tests-container:%=$(objpfx)%.out): $(objpfx)%.out : $(if $(wildcard $(objpfx)%.files),$(objpfx)%.files,/dev/null) $(objpfx)%
+	$(common-objpfx)support/test-container $(make-test-out) > $@; \
+	$(evaluate-test)
+
+
 # tests-unsupported lists tests that we will not try to build at all in
 # this configuration.  Note this runs every time because it does not
 # actually create its target.  The dependency on Makefile is meant to
diff --git a/support/Makefile b/support/Makefile
index 1bda81e55e..dc265e7585 100644
--- a/support/Makefile
+++ b/support/Makefile
@@ -145,6 +145,25 @@  ifeq ($(build-shared),yes)
 libsupport-inhibit-o += .o
 endif
 
+others: $(objpfx)test-container $(objpfx)links-dso-program
+
+$(objpfx)test-container : test-container.c
+	$(native-compile) -g \
+		-DSRCDIR_PATH=\"`cd .. ; pwd`\" \
+		-DOBJDIR_PATH=\"`cd $(objpfx)/..; pwd`\" \
+		-DINSTDIR_PATH=\"${prefix}\" \
+		-DLIBDIR_PATH=\"${libdir}\"
+
+# This exists only so we can guess which OS DSOs we need to copy into
+# the testing container.
+ifeq (,$(CXX))
+$(objpfx)links-dso-program : $(objpfx)links-dso-program-c.o
+	$(LINK.o) -o $@ $^
+else
+$(objpfx)links-dso-program : $(objpfx)links-dso-program.o
+	$(LINK.o) -o $@ $^ -lstdc++
+endif
+
 tests = \
   README-testing \
   tst-support-namespace \
diff --git a/support/links-dso-program-c.c b/support/links-dso-program-c.c
new file mode 100644
index 0000000000..c1f64fbbac
--- /dev/null
+++ b/support/links-dso-program-c.c
@@ -0,0 +1,4 @@ 
+int
+main() {
+  return 0;
+}
diff --git a/support/links-dso-program.cc b/support/links-dso-program.cc
new file mode 100644
index 0000000000..c1f64fbbac
--- /dev/null
+++ b/support/links-dso-program.cc
@@ -0,0 +1,4 @@ 
+int
+main() {
+  return 0;
+}
diff --git a/support/test-container.c b/support/test-container.c
new file mode 100644
index 0000000000..1f708f4527
--- /dev/null
+++ b/support/test-container.c
@@ -0,0 +1,1098 @@ 
+/* Run a test case in an isolated namespace.
+   Copyright (C) 2017-2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#define __USE_LARGEFILE64
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sched.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/capability.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <stdarg.h>
+#include <sys/sysmacros.h>
+#include <ctype.h>
+#include <utime.h>
+#include <errno.h>
+
+/* We can't use <libc-pointer-arith.h> because we're a native compile,
+   not a libc compile, so can only rely on the host's headers.  */
+#define ALIGN_UP(base, size) \
+  (((base) + (size) - 1) & -((__typeof__ (base)) (size)))
+
+#ifdef SRCDIR_PATH
+const char *srcdir = SRCDIR_PATH;
+#else
+# error please -DSRCDIR_PATH=something in the Makefile
+#endif
+
+#ifdef OBJDIR_PATH
+const char *objdir = OBJDIR_PATH;
+#else
+# error please -DOBJDIR_PATH=something in the Makefile
+#endif
+
+#ifdef INSTDIR_PATH
+const char *instdir = INSTDIR_PATH;
+#else
+# error please -DINSTDIR_PATH=something in the Makefile
+#endif
+
+#ifdef LIBDIR_PATH
+const char *libdir = LIBDIR_PATH;
+#else
+# error please -DLIBDIR_PATH=something in the Makefile
+#endif
+
+int verbose = 0;
+
+/* Running a test in a container is tricky.  There are two main
+   categories of things to do:
+
+   1. "Once" actions, like setting up the container and doing an
+      install into it.
+
+   2. "Per-test" actions, like copying in support files and
+      configuring the container.
+
+
+   "Once" actions:
+
+   * mkdir $buildroot/testroot.pristine/
+   * install into it
+   * rsync to $buildroot/testroot.root/
+
+   "Per-test" actions:
+   * maybe rsync to $buildroot/testroot.root/
+   * copy support files and test binary
+   * chroot/unshare
+   * set up any mounts (like /proc)
+
+   Magic files:
+
+   For test $srcdir/foo/mytest.c we look for $srcdir/foo/mytest.root and, if found...
+
+   * mytest.root/ is rsync'd into container
+   * mytest.root/preclean.txt causes fresh rsync (with delete) before test if present
+   * mytest.root/files.txt has a list of files to copy - TBD
+     ($B/ or $S/ for build and source directories)
+       syntax:
+         # comment
+         mv FILE FILE
+	 cp FILE FILE
+	 rm FILE
+	 FILE must start with $B/, $S/, $I/, $L/, or /
+	  (expands to build dir, source dir, install dir, library dir
+	   (in container), or container's root)
+   * mytest.root/postclean.txt causes fresh rsync (with delete) after test if present
+
+   Note that $srcdir/foo/mytest.files may be used instead of a
+   files.txt in the sysroot, if there is no other reason for a sysroot.
+
+   Design goals:
+
+   * independent of other packages which may not be installed (like
+     rsync or Docker, or even "cp")
+
+   * Simple, easy to review code (i.e. prefer simple naive code over
+     complex efficient code)
+
+   TBD:
+
+   * The current implementation is not parallel-make-safe, as one test
+     could be modifying the chroot while another is running against
+     it.  A simple lock is used to serialize tests.
+
+*/
+
+/*--------------------------------------------------*/
+/* Utility Functions */
+
+/* Temporarily concatenate multiple strings into one.  Allows up to 10
+   temporary results; use strdup () if you need them to be
+   permanent.  */
+
+static char *
+concat (const char *str, ...)
+{
+  /* Assume initialized to NULL/zero.  */
+  static char *bufs[10];
+  static size_t buflens[10];
+  static int bufn = 0;
+  int n;
+  size_t len;
+  va_list ap, ap2;
+  char *cp;
+  char *next;
+
+  va_start (ap, str);
+  va_copy (ap2, ap);
+
+  n = bufn;
+  bufn = (bufn + 1) % 10;
+  len = strlen (str);
+
+  while ((next = va_arg (ap, char *)) != NULL)
+    len = len + strlen (next);
+
+  va_end (ap);
+
+  if (bufs[n] == NULL)
+    {
+      bufs[n] = malloc (len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+  else if (buflens[n] < len + 1)
+    {
+      bufs[n] = realloc (bufs[n], len + 1); /* NUL */
+      buflens[n] = len + 1;
+    }
+
+  strcpy (bufs[n], str);
+  cp = strchr (bufs[n], '\0');
+  while ((next = va_arg (ap2, char *)) != NULL)
+    {
+      strcpy (cp, next);
+      cp = strchr (cp, '\0');
+    }
+  *cp = 0;
+  va_end (ap2);
+
+  return bufs[n];
+}
+
+/* Equivalent of "mkdir -p".  */
+
+static int
+mkdir_p (char *path)
+{
+  struct stat s;
+  char *slash_p;
+  int rv;
+
+  if (path[0] == 0)
+    return 0;
+
+  if (stat (path, &s) == 0)
+    {
+      if (S_ISDIR (s.st_mode))
+	return 0;
+      printf ("mkdir_p: %s exists but isn't a directory\n", path);
+      perror ("The error was");
+      exit (1);
+    }
+
+  slash_p = strrchr (path, '/');
+  if (slash_p)
+    {
+      while (slash_p > path && slash_p[-1] == '/')
+	--slash_p;
+      // Hack to allow top-level paths to be read-only and still work
+      if (slash_p > path)
+	{
+	  *slash_p = '\0';
+	  rv = mkdir_p (path);
+	  if (rv != 0)
+	    return rv;
+	  *slash_p = '/';
+	}
+    }
+
+  rv = mkdir (path, 0755);
+  if (rv != 0)
+    {
+      printf ("mkdir_p: can't create directory %s\n", path);
+      perror ("The error was");
+      return 1;
+    }
+
+  return 0;
+}
+
+/* Like the above, but creates the parent directory for file
+   FILEPATH.  */
+
+static int
+mkdir_parent_p (char *filepath)
+{
+  int rv;
+  char *slash_p = strrchr (filepath, '/');
+  if (slash_p)
+    {
+      while (slash_p > filepath && slash_p[-1] == '/')
+	--slash_p;
+      *slash_p = '\0';
+      rv = mkdir_p (filepath);
+      if (rv != 0)
+	return rv;
+      *slash_p = '/';
+    }
+  return 0;
+}
+
+/* Try to mount SRC onto DEST.  */
+
+static void
+trymount (const char *src, const char *dest)
+{
+  if (mount (src, dest, "", MS_BIND, NULL) < 0)
+    {
+      printf ("can't mount %s onto %s\n", src, dest);
+      perror ("the error was");
+      exit (1);
+    }
+}
+
+/* Special case of above for devices like /dev/zero where we have to
+   mount a device over a device, not a directory over a directory.  */
+
+static void
+devmount (const char *new_root_path, const char *which)
+{
+  int fd;
+  fd = open (concat (new_root_path, "/dev/", which, NULL),
+	     O_CREAT | O_TRUNC | O_RDWR, 0777);
+  close (fd);
+
+  trymount (concat ("/dev/", which, NULL),
+	    concat (new_root_path, "/dev/", which, NULL));
+}
+
+/* Returns true if the string "looks like" an environement variable
+   being set.  */
+
+static int
+is_env_setting (const char *a)
+{
+  int count_name = 0;
+
+  while (*a)
+    {
+      if (isalnum (*a) || *a == '_')
+	++count_name;
+      else if (*a == '=' && count_name > 0)
+	return 1;
+      else
+	return 0;
+      ++a;
+    }
+  return 0;
+}
+
+/* Break the_line into words and store in the_words.  Max nwords,
+   returns actual count.  */
+static int
+tokenize (char *the_line, char **the_words, int nwords)
+{
+  int rv = 0;
+
+  while (nwords > 0)
+    {
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && isspace (*the_line))
+	++the_line;
+
+      /* End of line?  */
+      if (*the_line == 0)
+	return rv;
+
+      /* THE_LINE points to a non-whitespace character, so we have a
+	 word.  */
+      *the_words = the_line;
+      ++the_words;
+      nwords--;
+      ++rv;
+
+      /* Skip leading whitespace, if any.  */
+      while (*the_line && ! isspace (*the_line))
+	++the_line;
+
+      /* We now point at the trailing NUL *or* some whitespace.  */
+      if (*the_line == 0)
+	return rv;
+
+      /* It was whitespace, skip and keep tokenizing.  */
+      *the_line++ = 0;
+    }
+
+  /* We get here if we filled the words buffer.  */
+  return rv;
+}
+
+/*--------------------------------------------------*/
+/* mini-RSYNC implementation.  Optimize later.      */
+
+/* Set this to 1 if you need to debug the rsync function.  */
+#define RTRACE 0
+
+/* A few routines for an "rsync buffer" which stores the paths we're
+   working on.  We continuously grow and shrink the paths in each
+   buffer so there's lot of re-use.  */
+
+/* We rely on "initialized to zero" to set these up.  */
+typedef struct
+{
+  char *buf;
+  size_t len;
+  size_t size;
+} path_buf;
+
+static path_buf spath, dpath;
+
+static void
+r_setup (char *path, path_buf * pb)
+{
+  size_t len = strlen (path);
+  if (pb->buf == NULL || pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      if (pb->buf == NULL)
+	pb->buf = (char *) malloc (sz);
+      else
+	pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	{
+	  printf ("Out of memory while rsyncing\n");
+	  exit (1);
+	}
+      pb->size = sz;
+    }
+  strcpy (pb->buf, path);
+  pb->len = len;
+}
+
+static void
+r_append (char *path, path_buf * pb)
+{
+  size_t len = strlen (path) + pb->len;
+  if (pb->size < len + 1)
+    {
+      /* Round up */
+      size_t sz = ALIGN_UP (len + 1, 512);
+      pb->buf = (char *) realloc (pb->buf, sz);
+      if (pb->buf == NULL)
+	{
+	  printf ("Out of memory while rsyncing\n");
+	  exit (1);
+	}
+      pb->size = sz;
+    }
+  strcpy (pb->buf + pb->len, path);
+  pb->len = len;
+}
+
+static int
+file_exists (char *path)
+{
+  struct stat st;
+  if (lstat (path, &st) == 0)
+    return 1;
+  return 0;
+}
+
+static void
+recursive_remove (char *path)
+{
+  pid_t child;
+  int status;
+  /* FIXME: re-implement without external dependencies at some point.
+     Fortunately, this runs outside the container.  */
+
+  child = fork();
+
+  switch (child) {
+  case -1:
+    perror ("fork");
+    exit (1);
+  case 0:
+    /* Child.  */
+    execlp ("rm", "rm", "-rf", path, NULL);
+  default:
+    /* Parent.  */
+    waitpid (child, &status, 0);
+    break;
+  }
+}
+
+/* Used for both rsync and the files.txt "cp" command.  */
+
+static void
+copy_one_file (const char *sname, const char *dname)
+{
+  int sfd, dfd;
+  char buf[512];
+  size_t rsz, wsz;
+  struct stat st;
+  struct utimbuf times;
+
+  sfd = open (sname, O_RDONLY);
+  if (sfd < 0)
+    {
+      printf ("unable to open %s for reading\n", sname);
+      perror ("the error was");
+      exit (1);
+    }
+  dfd = open (dname, O_WRONLY | O_TRUNC | O_CREAT, 0600);
+  if (dfd < 0)
+    {
+      printf ("unable to open %s for writing\n", dname);
+      perror ("the error was");
+      exit (1);
+    }
+
+  while ((rsz = read (sfd, buf, 512)) > 0)
+    write (dfd, buf, rsz);
+
+  if (rsz < 0)
+    {
+      printf ("error reading from %s\n", sname);
+      perror ("the error was");
+      exit (1);
+    }
+
+  close (sfd);
+  close (dfd);
+
+  stat (sname, &st);
+  chmod (dname, st.st_mode & 0777);
+
+  times.actime = st.st_atime;
+  times.modtime = st.st_mtime;
+  utime (dname, &times);
+}
+
+/* We don't check *everything* about the two files to see if a copy is
+   needed, just the minimum to make sure we get the latest copy. */
+static int
+need_sync (char *ap, char *bp, struct stat *a, struct stat *b)
+{
+  if (a->st_mode & S_IFMT != b->st_mode & S_IFMT)
+    return 1;
+
+  if (S_ISLNK (a->st_mode))
+    {
+      int rv;
+      char *al, *bl;
+
+      if (a->st_size != b->st_size)
+	return 1;
+
+      al = (char *) malloc (a->st_size + 1);
+      bl = (char *) malloc (b->st_size + 1);
+      readlink (ap, al, a->st_size + 1);
+      readlink (bp, bl, b->st_size + 1);
+      al[a->st_size] = 0;
+      bl[b->st_size] = 0;
+      rv = strcmp (al, bl);
+      free (al);
+      free (bl);
+      if (rv == 0)
+	return 0; /* links are same */
+      return 1; /* links differ */
+    }
+
+#if RTRACE
+  if (a->st_size != b->st_size)
+    printf ("SIZE\n");
+  if ((a->st_mode & 0777) != (b->st_mode & 0777))
+    printf ("MODE\n");
+  if (a->st_mtime != b->st_mtime)
+    printf ("TIME\n");
+#endif
+
+  if (a->st_size == b->st_size
+      && ((a->st_mode & 0777) == (b->st_mode & 0777))
+      && a->st_mtime == b->st_mtime)
+    return 0;
+
+  return 1;
+}
+
+static void
+rsync_1 (path_buf * src, path_buf * dest, int and_delete)
+{
+  DIR *dir;
+  struct dirent *de;
+  struct stat s, d;
+
+  r_append ("/", src);
+  r_append ("/", dest);
+#if RTRACE
+  printf ("sync %s to %s %s\n", src->buf, dest->buf,
+	  and_delete ? "and delete" : "");
+#endif
+
+  size_t staillen = src->len;
+  char *stail = src->buf + src->len;
+
+  size_t dtaillen = dest->len;
+  char *dtail = dest->buf + dest->len;
+
+  dir = opendir (src->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      lstat (dest->buf, &d);
+
+      if (s.st_mode == ~0)
+	{
+	  printf ("Error: %s obtained by readdir, but stat failed.\n",
+		  src->buf);
+	  exit (1);
+	}
+
+      if (! need_sync (src->buf, dest->buf, &s, &d))
+	{
+	  if (S_ISDIR (s.st_mode))
+	    rsync_1 (src, dest, and_delete);
+	  continue;
+	}
+
+      if (d.st_mode != ~0)
+	switch (d.st_mode & S_IFMT)
+	  {
+	  case S_IFDIR:
+	    if (!S_ISDIR (s.st_mode))
+	      {
+#if RTRACE
+		printf ("-D %s\n", dest->buf);
+#endif
+		recursive_remove (dest->buf);
+	      }
+	    break;
+
+	  default:
+#if RTRACE
+	    printf ("-F %s\n", dest->buf);
+#endif
+	    unlink (dest->buf);
+	    break;
+	  }
+
+      switch (s.st_mode & S_IFMT)
+	{
+	case S_IFREG:
+#if RTRACE
+	  printf ("+F %s\n", dest->buf);
+#endif
+	  copy_one_file (src->buf, dest->buf);
+	  break;
+
+	case S_IFDIR:
+#if RTRACE
+	  printf ("+D %s\n", dest->buf);
+#endif
+	  mkdir (dest->buf, (s.st_mode & 0777) | 0700);
+	  rsync_1 (src, dest, and_delete);
+	  break;
+
+	case S_IFLNK:
+	  {
+	    char *lp = (char *) malloc (s.st_size + 1);
+#if RTRACE
+	    printf ("+L %s\n", dest->buf);
+#endif
+	    readlink (src->buf, lp, s.st_size + 1);
+	    lp[s.st_size] = 0;
+	    symlink (lp, dest->buf);
+	    free (lp);
+	    break;
+	  }
+
+	default:
+	  break;
+	}
+    }
+
+  closedir (dir);
+  src->len = staillen;
+  src->buf[staillen] = 0;
+  dest->len = dtaillen;
+  dest->buf[dtaillen] = 0;
+
+  if (!and_delete)
+    return;
+
+  /* The rest of this function removes any files/directories in DEST
+     that do not exist in SRC.  This is triggered as part of a
+     preclean or postsclean step.  */
+
+  dir = opendir (dest->buf);
+
+  while ((de = readdir (dir)) != NULL)
+    {
+      if (strcmp (de->d_name, ".") == 0
+	  || strcmp (de->d_name, "..") == 0)
+	continue;
+
+      src->len = staillen;
+      r_append (de->d_name, src);
+      dest->len = dtaillen;
+      r_append (de->d_name, dest);
+
+      s.st_mode = ~0;
+      d.st_mode = ~0;
+
+      lstat (src->buf, &s);
+      lstat (dest->buf, &d);
+
+      if (d.st_mode == ~0)
+	{
+	  printf ("Error: %s obtained by readdir, but stat failed.\n",
+		  dest->buf);
+	  exit (1);
+	}
+      if (s.st_mode == ~0)
+	{
+	  /* dest exists and src doesn't, clean it.  */
+	  switch (d.st_mode & S_IFMT)
+	    {
+	    case S_IFDIR:
+	      if (!S_ISDIR (s.st_mode))
+		{
+#if RTRACE
+		  printf ("-D %s\n", dest->buf);
+#endif
+		  recursive_remove (dest->buf);
+		}
+	      break;
+
+	    default:
+#if RTRACE
+	      printf ("-F %s\n", dest->buf);
+#endif
+	      unlink (dest->buf);
+	      break;
+	    }
+	}
+    }
+
+  closedir (dir);
+}
+
+static void
+rsync (char *src, char *dest, int and_delete)
+{
+  r_setup (src, &spath);
+  r_setup (dest, &dpath);
+
+  rsync_1 (&spath, &dpath, and_delete);
+}
+
+/*--------------------------------------------------*/
+/* Main */
+
+int
+main (int argc, char **argv)
+{
+  pid_t child;
+  char *pristine_root_path;
+  char *new_root_path;
+  char *new_cwd_path;
+  char *new_objdir_path;
+  char *new_srcdir_path;
+  char **new_child_proc;
+  char *command_root;
+  char *command_base;
+  char *so_base;
+  int do_postclean = 0;
+
+  uid_t original_uid;
+  gid_t original_gid;
+  int UMAP;
+  int GMAP;
+  char tmp[100];
+  struct stat st;
+  int lock_fd;
+
+  setbuf (stdout, NULL);
+
+  // The command line we're expecting looks like this:
+  // env <set some vars> ld.so <library path> test-binary
+
+  // We need to peel off any "env" or "ld.so" portion of the command
+  // line, and keep track of which env vars we should preserve and
+  // which we drop.
+
+  if (argc < 2)
+    {
+      fprintf (stderr, "Usage: containerize <program to run> <args...>\n");
+      exit (1);
+    }
+
+  if (strcmp (argv[1], "-v") == 0)
+    {
+      verbose = 1;
+      ++argv;
+      --argc;
+    }
+
+  if (strcmp (argv[1], "env") == 0)
+    {
+      ++argv;
+      --argc;
+      while (is_env_setting (argv[1]))
+	{
+	  // List variables we do NOT want to propogate.
+#if 0
+	  // until we discover why locale/iconv tests don't
+	  // work against an installed tree...
+	  if (memcmp (argv[1], "GCONV_PATH=", 11)
+	      && memcmp (argv[1], "LOCPATH=", 8))
+#endif
+	    {
+	      // Need to keep these.  Note that putenv stores a
+	      // pointer to our argv.
+	      putenv (argv[1]);
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  if (strncmp (argv[1], concat (objdir, "/elf/ld-linux-", NULL),
+	       strlen (objdir) + 14) == 0)
+    {
+      ++argv;
+      --argc;
+      while (argv[1][0] == '-')
+	{
+	  if (strcmp (argv[1], "--library-path") == 0)
+	    {
+	      ++argv;
+	      --argc;
+	    }
+	  ++argv;
+	  --argc;
+	}
+    }
+
+  pristine_root_path = strdup (concat (objdir, "/testroot.pristine", NULL));
+  new_root_path = strdup (concat (objdir, "/testroot.root", NULL));
+  new_cwd_path = get_current_dir_name ();
+  new_child_proc = argv + 1;
+
+  lock_fd = open(concat (pristine_root_path, "/lock.fd", NULL), O_CREAT | O_TRUNC | O_RDWR, 0666);
+  if (lock_fd < 0)
+    {
+      printf ("Cannot create testroot lock.\n");
+      perror("The error was");
+      exit (77);
+    }
+  while (flock (lock_fd, LOCK_EX) != 0)
+    {
+      if (errno != EINTR)
+	{
+	  printf ("Cannot lock testroot.\n");
+	  perror("The error was");
+	  exit (77);
+	}
+    }
+
+  mkdir_p (new_root_path);
+
+  /* We look for extra setup info in a subdir in the same spot as the
+     test, with the same name but a ".root" extension.  This is that
+     directory.  We try to look in the source tree if the path we're
+     given refers to the build tree, but we rely on the path to be
+     absolute.  This is what the glibc makefiles do.  */
+  command_root = concat (argv[1], ".root", NULL);
+  if (strncmp (command_root, objdir, strlen (objdir)) == 0
+      && command_root[strlen (objdir)] == '/')
+    command_root = concat (srcdir, argv[1] + strlen (objdir), ".root", NULL);
+  command_root = strdup (command_root);
+
+  /* This cuts off the ".root" we appended above.  */
+  command_base = strdup (command_root);
+  command_base[strlen (command_base) - 5] = 0;
+
+  /* Shared object base directory.  */
+  so_base = strdup (argv[1]);
+  if (strrchr (so_base, '/') != NULL)
+    strrchr (so_base, '/')[1] = 0;
+
+  if (file_exists (concat (command_root, "/postclean.txt", NULL)))
+    do_postclean = 1;
+
+  rsync (pristine_root_path, new_root_path,
+	 1 || file_exists (concat (command_root, "/preclean.txt", NULL)));
+
+  if (stat (command_root, &st) >= 0
+      && S_ISDIR (st.st_mode))
+    rsync (command_root, new_root_path, 0);
+
+  new_objdir_path = strdup (concat (new_root_path, objdir, NULL));
+  new_srcdir_path = strdup (concat (new_root_path, srcdir, NULL));
+
+  /* new_cwd_path starts with '/' so no "/" needed between the two.  */
+  mkdir_p (concat (new_root_path, new_cwd_path, NULL));
+  mkdir_p (new_srcdir_path);
+  mkdir_p (new_objdir_path);
+
+  original_uid = getuid ();
+  original_gid = getgid ();
+
+  /* Handle the cp/mv/rm "script" here.  */
+  {
+    char *the_line = NULL;
+    size_t line_len = 0;
+    char *fname = concat (command_root, "/files.txt", NULL);
+    char *the_words[3];
+    FILE *f = fopen (fname, "r");
+
+    if (verbose && f)
+      fprintf (stderr, "reading %s\n", fname);
+
+    if (f == NULL)
+      {
+	/* Try foo.files instead of foo.root/files.txt, as a shortcut.  */
+	fname = concat (command_base, ".files", NULL);
+	f = fopen (fname, "r");
+	if (verbose && f)
+	  fprintf (stderr, "reading %s\n", fname);
+      }
+
+#if 0
+    /* I don't want to add this until we know we need it, but here it
+       is...  */
+    if (f == NULL)
+      {
+	/* Look for a Makefile-generated one also.  */
+	fname = concat (argv[1], ".files", NULL);
+	f = fopen (fname, "r");
+      }
+#endif
+
+    /* This is where we "interpret" the mini-script which is <test>.files.  */
+    if (f != NULL)
+      {
+	while (getline (&the_line, &line_len, f) > 0)
+	  {
+	    int nt = tokenize (the_line, the_words, 3);
+	    int i;
+
+	    for (i = 1; i < nt; ++i)
+	      {
+		if (memcmp (the_words[i], "$B/", 3) == 0)
+		  the_words[i] = concat (objdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$S/", 3) == 0)
+		  the_words[i] = concat (srcdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$I/", 3) == 0)
+		  the_words[i] = concat (new_root_path, instdir, the_words[i] + 2, NULL);
+		else if (memcmp (the_words[i], "$L/", 3) == 0)
+		  the_words[i] = concat (new_root_path, libdir, the_words[i] + 2, NULL);
+		else if (the_words[i][0] == '/')
+		  the_words[i] = concat (new_root_path, the_words[i], NULL);
+	      }
+
+	    if (nt == 3 && the_words[2][strlen (the_words[2]) - 1] == '/')
+	      {
+		char *r = strrchr (the_words[1], '/');
+		if (r)
+		  the_words[2] = concat (the_words[2], r + 1, NULL);
+		else
+		  the_words[2] = concat (the_words[2], the_words[1], NULL);
+	      }
+
+	    if (nt == 2 && strcmp (the_words[0], "so") == 0)
+	      {
+		the_words[2] = concat (new_root_path, libdir, "/", the_words[1], NULL);
+		the_words[1] = concat (so_base, the_words[1], NULL);
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "cp") == 0)
+	      {
+		copy_one_file (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "mv") == 0)
+	      {
+		rename (the_words[1], the_words[2]);
+	      }
+	    else if (nt == 3 && strcmp (the_words[0], "chmod") == 0)
+	      {
+		long int m;
+		m = strtol (the_words[1], NULL, 0);
+		chmod (the_words[2], m);
+	      }
+	    else if (nt == 2 && strcmp (the_words[0], "rm") == 0)
+	      {
+		unlink (the_words[1]);
+	      }
+	    else if (nt > 0 && the_words[0][0] != '#')
+	      {
+		printf ("\033[31minvalid [%s]\033[0m\n", the_words[0]);
+	      }
+	  }
+	fclose (f);
+      }
+  }
+
+  // The unshare here gives us our own spaces and capabilities.
+  if (unshare (CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS) < 0)
+    {
+      printf ("unable to unshare user/fs, ");
+      perror ("the error was");
+      exit (1);
+    }
+
+  /* Some systems, by default, all mounts leak out of the namespace.  */
+  if (mount ("none", "/", NULL, MS_REC | MS_PRIVATE, NULL) != 0)
+    {
+      printf ("warning: could not create a private mount namespace: %m\n");
+      exit (1);
+    }
+
+  trymount (srcdir, new_srcdir_path);
+  trymount (objdir, new_objdir_path);
+
+  mkdir_p (concat (new_root_path, "/dev", NULL));
+  devmount (new_root_path, "null");
+  devmount (new_root_path, "zero");
+  devmount (new_root_path, "urandom");
+
+  // We're done with the "old" root, switch to the new one.
+  if (chroot (new_root_path) < 0)
+    {
+      printf ("Can't chroot to %s - ", new_root_path);
+      perror ("the error was");
+      exit (1);
+    }
+
+  if (chdir (new_cwd_path) < 0)
+    {
+      printf ("Can't cd to new %s - ", new_cwd_path);
+      perror ("the error was");
+      exit (1);
+    }
+
+  /* To complete the containerization, we need to fork () at least
+     once.  We can't exec, nor can we somehow link the new child to
+     our parent.  So we run the child and propogate it's exit status
+     up. */
+  child = fork ();
+  if (child < 0)
+    {
+      perror ("fork");
+      exit (1);
+    }
+  else if (child > 0)
+    {
+      /* Parent.  */
+      int status;
+      waitpid (child, &status, 0);
+
+      /* There's a bit of magic here, since the buildroot is mounted
+	 in our space, the paths are still valid, and since the mounts
+	 aren't recursive, it sees *only* the built root, not anything
+	 we would normally se if we rsync'd to "/" like mounted /dev
+	 files.  */
+      if (do_postclean)
+	  rsync (pristine_root_path, new_root_path, 1);
+
+      if (WIFEXITED (status))
+	exit (WEXITSTATUS (status));
+
+      if (WIFSIGNALED (status))
+	{
+	  printf ("%%SIGNALLED%%\n");
+	  exit (77);
+	}
+
+      printf ("%%EXITERROR%%\n");
+      exit (78);
+    }
+
+  /* The rest is the child process, which is now PID 1 and "in" the
+     new root.  */
+
+  mkdir_p ("/tmp");
+
+  // Now that we're pid 1 (effectively "root") we can mount /proc
+  mkdir ("/proc", 0777);
+  if (mount ("proc", "/proc", "proc", 0, NULL) < 0)
+    {
+      printf ("Unable to mount /proc: ");
+      perror ("the error was");
+    }
+
+  // We map our original UID to the same UID in the container so we
+  // can own our own files normally
+  UMAP = open ("/proc/self/uid_map", O_WRONLY);
+  if (UMAP < 0)
+    {
+      printf ("can't write to /proc/self/uid_map\n");
+      perror ("The error was");
+      exit (1);
+    }
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_uid, (long long) original_uid);
+  write (UMAP, tmp, strlen (tmp));
+  close (UMAP);
+
+  // We must disable setgroups () before we can map our groups, else we
+  // get EPERM.
+  GMAP = open ("/proc/self/setgroups", O_WRONLY);
+  if (GMAP >= 0)
+    {
+      /* We support kernels old enough to not have this.  */
+      write (GMAP, "deny\n", 5);
+      close (GMAP);
+    }
+
+  // We map our original GID to the same GID in the container so we
+  // can own our own files normally
+  GMAP = open ("/proc/self/gid_map", O_WRONLY);
+  if (GMAP < 0)
+    {
+      printf ("can't write to /proc/self/gid_map\n");
+      perror ("The error was");
+      exit (1);
+    }
+  sprintf (tmp, "%lld %lld 1\n", (long long) original_gid, (long long) original_gid);
+  write (GMAP, tmp, strlen (tmp));
+  close (GMAP);
+
+  // Now run the child
+  execvp (new_child_proc[0], new_child_proc);
+
+  // Or don't run the child?
+  printf ("Unable to exec %s\n", new_child_proc[0]);
+  perror ("The error was");
+  exit (77);
+}