@@ -35,7 +35,7 @@
#include <dlfcn.h>
#endif
-#define omp_max_predefined_alloc omp_thread_mem_alloc
+#define omp_max_predefined_alloc ompx_pinned_mem_alloc
/* These macros may be overridden in config/<target>/allocator.c.
The following definitions (ab)use comma operators to avoid unused
@@ -76,6 +76,7 @@ static const omp_memspace_handle_t predefined_alloc_mapping[] = {
omp_low_lat_mem_space, /* omp_cgroup_mem_alloc. */
omp_low_lat_mem_space, /* omp_pteam_mem_alloc. */
omp_low_lat_mem_space, /* omp_thread_mem_alloc. */
+ omp_default_mem_space, /* ompx_pinned_mem_alloc. */
};
enum gomp_numa_memkind_kind
@@ -612,8 +613,10 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- ptr = MEMSPACE_ALLOC (memspace, new_size,
- allocator_data && allocator_data->pinned);
+ int pinned = (allocator_data
+ ? allocator_data->pinned
+ : allocator == ompx_pinned_mem_alloc);
+ ptr = MEMSPACE_ALLOC (memspace, new_size, pinned);
}
if (ptr == NULL)
goto fail;
@@ -634,7 +637,8 @@ retry:
fail:;
int fallback = (allocator_data
? allocator_data->fallback
- : allocator == omp_default_mem_alloc
+ : (allocator == omp_default_mem_alloc
+ || allocator == ompx_pinned_mem_alloc)
? omp_atv_null_fb
: omp_atv_default_mem_fb);
switch (fallback)
@@ -762,6 +766,7 @@ omp_free (void *ptr, omp_allocator_handle_t allocator)
#endif
memspace = predefined_alloc_mapping[data->allocator];
+ pinned = (data->allocator == ompx_pinned_mem_alloc);
}
MEMSPACE_FREE (memspace, data->ptr, data->size, pinned);
@@ -935,8 +940,10 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- ptr = MEMSPACE_CALLOC (memspace, new_size,
- allocator_data && allocator_data->pinned);
+ int pinned = (allocator_data
+ ? allocator_data->pinned
+ : allocator == ompx_pinned_mem_alloc);
+ ptr = MEMSPACE_CALLOC (memspace, new_size, pinned);
}
if (ptr == NULL)
goto fail;
@@ -957,7 +964,8 @@ retry:
fail:;
int fallback = (allocator_data
? allocator_data->fallback
- : allocator == omp_default_mem_alloc
+ : (allocator == omp_default_mem_alloc
+ || allocator == ompx_pinned_mem_alloc)
? omp_atv_null_fb
: omp_atv_default_mem_fb);
switch (fallback)
@@ -1180,11 +1188,14 @@ retry:
else
#endif
if (prev_size)
- new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
- data->size, new_size,
- (free_allocator_data
- && free_allocator_data->pinned),
- allocator_data->pinned);
+ {
+ int was_pinned = (free_allocator_data
+ ? free_allocator_data->pinned
+ : free_allocator == ompx_pinned_mem_alloc);
+ new_ptr = MEMSPACE_REALLOC (allocator_data->memspace, data->ptr,
+ data->size, new_size, was_pinned,
+ allocator_data->pinned);
+ }
else
new_ptr = MEMSPACE_ALLOC (allocator_data->memspace, new_size,
allocator_data->pinned);
@@ -1240,10 +1251,14 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
+ int was_pinned = (free_allocator_data
+ ? free_allocator_data->pinned
+ : free_allocator == ompx_pinned_mem_alloc);
+ int pinned = (allocator_data
+ ? allocator_data->pinned
+ : allocator == ompx_pinned_mem_alloc);
new_ptr = MEMSPACE_REALLOC (memspace, data->ptr, data->size, new_size,
- (free_allocator_data
- && free_allocator_data->pinned),
- allocator_data && allocator_data->pinned);
+ was_pinned, pinned);
}
if (new_ptr == NULL)
goto fail;
@@ -1277,8 +1292,10 @@ retry:
memspace = (allocator_data
? allocator_data->memspace
: predefined_alloc_mapping[allocator]);
- new_ptr = MEMSPACE_ALLOC (memspace, new_size,
- allocator_data && allocator_data->pinned);
+ int pinned = (allocator_data
+ ? allocator_data->pinned
+ : allocator == ompx_pinned_mem_alloc);
+ new_ptr = MEMSPACE_ALLOC (memspace, new_size, pinned);
}
if (new_ptr == NULL)
goto fail;
@@ -1333,7 +1350,9 @@ retry:
was_memspace = (free_allocator_data
? free_allocator_data->memspace
: predefined_alloc_mapping[free_allocator]);
- int was_pinned = (free_allocator_data && free_allocator_data->pinned);
+ int was_pinned = (free_allocator_data
+ ? free_allocator_data->pinned
+ : free_allocator == ompx_pinned_mem_alloc);
MEMSPACE_FREE (was_memspace, data->ptr, data->size, was_pinned);
}
return ret;
@@ -1341,7 +1360,8 @@ retry:
fail:;
int fallback = (allocator_data
? allocator_data->fallback
- : allocator == omp_default_mem_alloc
+ : (allocator == omp_default_mem_alloc
+ || allocator == ompx_pinned_mem_alloc)
? omp_atv_null_fb
: omp_atv_default_mem_fb);
switch (fallback)
@@ -134,6 +134,7 @@ typedef enum omp_allocator_handle_t __GOMP_UINTPTR_T_ENUM
omp_cgroup_mem_alloc = 6,
omp_pteam_mem_alloc = 7,
omp_thread_mem_alloc = 8,
+ ompx_pinned_mem_alloc = 9,
__omp_allocator_handle_t_max__ = __UINTPTR_MAX__
} omp_allocator_handle_t;
@@ -158,6 +158,8 @@
parameter :: omp_pteam_mem_alloc = 7
integer (kind=omp_allocator_handle_kind), &
parameter :: omp_thread_mem_alloc = 8
+ integer (kind=omp_allocator_handle_kind), &
+ parameter :: ompx_pinned_mem_alloc = 9
integer (omp_memspace_handle_kind), &
parameter :: omp_default_mem_space = 0
integer (omp_memspace_handle_kind), &
new file mode 100644
@@ -0,0 +1,103 @@
+/* { dg-do run } */
+
+/* { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } } */
+
+/* Test that ompx_pinned_mem_alloc works. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+#define CHECK_SIZE(SIZE) { \
+ struct rlimit limit; \
+ if (getrlimit (RLIMIT_MEMLOCK, &limit) \
+ || limit.rlim_cur <= SIZE) \
+ fprintf (stderr, "unsufficient lockable memory; please increase ulimit\n"); \
+ }
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+#else
+#define PAGE_SIZE 1 /* unknown */
+#define CHECK_SIZE(SIZE) fprintf (stderr, "OS unsupported\n");
+
+int
+get_pinned_mem ()
+{
+ return 0;
+}
+#endif
+
+static void
+verify0 (char *p, size_t s)
+{
+ for (size_t i = 0; i < s; ++i)
+ if (p[i] != 0)
+ abort ();
+}
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* Allocate at least a page each time, allowing space for overhead,
+ but stay within the ulimit. */
+ const int SIZE = PAGE_SIZE - 128;
+ CHECK_SIZE (SIZE * 5);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ void *p = omp_alloc (SIZE, ompx_pinned_mem_alloc);
+ if (!p)
+ abort ();
+
+ int amount = get_pinned_mem ();
+ if (amount == 0)
+ abort ();
+
+ p = omp_realloc (p, SIZE * 2, ompx_pinned_mem_alloc, ompx_pinned_mem_alloc);
+
+ int amount2 = get_pinned_mem ();
+ if (amount2 <= amount)
+ abort ();
+
+ /* SIZE*2 ensures that it doesn't slot into the space possibly
+ vacated by realloc. */
+ p = omp_calloc (1, SIZE * 2, ompx_pinned_mem_alloc);
+
+ if (get_pinned_mem () <= amount2)
+ abort ();
+
+ verify0 (p, SIZE * 2);
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,101 @@
+/* { dg-do run } */
+
+/* Test that ompx_pinned_mem_alloc fails correctly. */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef __linux__
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sys/mman.h>
+#include <sys/resource.h>
+
+#define PAGE_SIZE sysconf(_SC_PAGESIZE)
+
+int
+get_pinned_mem ()
+{
+ int pid = getpid ();
+ char buf[100];
+ sprintf (buf, "/proc/%d/status", pid);
+
+ FILE *proc = fopen (buf, "r");
+ if (!proc)
+ abort ();
+ while (fgets (buf, 100, proc))
+ {
+ int val;
+ if (sscanf (buf, "VmLck: %d", &val))
+ {
+ fclose (proc);
+ return val;
+ }
+ }
+ abort ();
+}
+
+void
+set_pin_limit (int size)
+{
+ struct rlimit limit;
+ if (getrlimit (RLIMIT_MEMLOCK, &limit))
+ abort ();
+ limit.rlim_cur = (limit.rlim_max < size ? limit.rlim_max : size);
+ if (setrlimit (RLIMIT_MEMLOCK, &limit))
+ abort ();
+}
+#else
+#define PAGE_SIZE 10000 * 1024 /* unknown */
+
+int
+get_pinned_mem ()
+{
+ return 0;
+}
+
+void
+set_pin_limit ()
+{
+}
+#endif
+
+#include <omp.h>
+
+int
+main ()
+{
+ /* Allocate at least a page each time, but stay within the ulimit. */
+ const int SIZE = PAGE_SIZE * 4;
+
+ /* Ensure that the limit is smaller than the allocation. */
+ set_pin_limit (SIZE / 2);
+
+ // Sanity check
+ if (get_pinned_mem () != 0)
+ abort ();
+
+ // Should fail
+ void *p = omp_alloc (SIZE, ompx_pinned_mem_alloc);
+ if (p)
+ abort ();
+
+ // Should fail
+ p = omp_calloc (1, SIZE, ompx_pinned_mem_alloc);
+ if (p)
+ abort ();
+
+ // Should fail to realloc
+ void *notpinned = omp_alloc (SIZE, omp_default_mem_alloc);
+ p = omp_realloc (notpinned, SIZE, ompx_pinned_mem_alloc, omp_default_mem_alloc);
+ if (!notpinned || p)
+ abort ();
+
+ // No memory should have been pinned
+ int amount = get_pinned_mem ();
+ if (amount != 0)
+ abort ();
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,16 @@
+! Ensure that the ompx_pinned_mem_alloc predefined allocator is present and
+! accepted. The majority of the functionality testing lives in the C tests.
+!
+! { dg-xfail-run-if "Pinning not implemented on this host" { ! *-*-linux-gnu } }
+
+program main
+ use omp_lib
+ use ISO_C_Binding
+ implicit none (external, type)
+
+ type(c_ptr) :: p
+
+ p = omp_alloc (10_c_size_t, ompx_pinned_mem_alloc);
+ if (.not. c_associated (p)) stop 1
+ call omp_free (p, ompx_pinned_mem_alloc);
+end program main