@@ -2616,6 +2616,7 @@ GTFILES = $(CPPLIB_H) $(srcdir)/input.h
$(srcdir)/omp-offload.h \
$(srcdir)/omp-offload.c \
$(srcdir)/omp-expand.c \
+ $(srcdir)/omp-general.c \
$(srcdir)/omp-low.c \
$(srcdir)/targhooks.c $(out_file) $(srcdir)/passes.c $(srcdir)/cgraphunit.c \
$(srcdir)/cgraphclones.c \
@@ -937,7 +937,8 @@ struct GTY((tag ("SYMTAB_FUNCTION"))) cg
split_part (false), indirect_call_target (false), local (false),
versionable (false), can_change_signature (false),
redefined_extern_inline (false), tm_may_enter_irr (false),
- ipcp_clone (false), m_uid (uid), m_summary_id (-1)
+ ipcp_clone (false), declare_variant_alt (false),
+ calls_declare_variant_alt (false), m_uid (uid), m_summary_id (-1)
{}
/* Remove the node from cgraph and all inline clones inlined into it.
@@ -1539,6 +1540,11 @@ struct GTY((tag ("SYMTAB_FUNCTION"))) cg
unsigned tm_may_enter_irr : 1;
/* True if this was a clone created by ipa-cp. */
unsigned ipcp_clone : 1;
+ /* True if this is the deferred declare variant resolution artificial
+ function. */
+ unsigned declare_variant_alt : 1;
+ /* True if the function calls declare_variant_alt functions. */
+ unsigned calls_declare_variant_alt : 1;
private:
/* Unique id of the node. */
@@ -450,6 +450,9 @@ symbol_table::remove_unreachable_nodes (
reachable.add (body);
reachable.add (e->callee);
}
+ else if (e->callee->declare_variant_alt
+ && !e->callee->in_other_partition)
+ reachable.add (e->callee);
enqueue_node (e->callee, &first, &reachable);
}
@@ -535,6 +535,8 @@ lto_output_node (struct lto_simple_outpu
bp_pack_value (&bp, node->merged_extern_inline, 1);
bp_pack_value (&bp, node->thunk.thunk_p, 1);
bp_pack_value (&bp, node->parallelized_function, 1);
+ bp_pack_value (&bp, node->declare_variant_alt, 1);
+ bp_pack_value (&bp, node->calls_declare_variant_alt, 1);
bp_pack_enum (&bp, ld_plugin_symbol_resolution,
LDPR_NUM_KNOWN,
/* When doing incremental link, we will get new resolution
@@ -1186,6 +1188,8 @@ input_overwrite_node (struct lto_file_de
node->merged_extern_inline = bp_unpack_value (bp, 1);
node->thunk.thunk_p = bp_unpack_value (bp, 1);
node->parallelized_function = bp_unpack_value (bp, 1);
+ node->declare_variant_alt = bp_unpack_value (bp, 1);
+ node->calls_declare_variant_alt = bp_unpack_value (bp, 1);
node->resolution = bp_unpack_enum (bp, ld_plugin_symbol_resolution,
LDPR_NUM_KNOWN);
node->split_part = bp_unpack_value (bp, 1);
@@ -477,6 +477,7 @@ simd_clone_create (struct cgraph_node *o
the old node. */
new_node->local = old_node->local;
new_node->externally_visible = old_node->externally_visible;
+ new_node->calls_declare_variant_alt = old_node->calls_declare_variant_alt;
return new_node;
}
@@ -4900,6 +4900,8 @@ expand_call_inline (basic_block bb, gimp
if (src_properties != prop_mask)
dst_cfun->curr_properties &= src_properties | ~prop_mask;
dst_cfun->calls_eh_return |= id->src_cfun->calls_eh_return;
+ id->dst_node->calls_declare_variant_alt
+ |= id->src_node->calls_declare_variant_alt;
gcc_assert (!id->src_cfun->after_inlining);
@@ -6231,6 +6233,8 @@ tree_function_versioning (tree old_decl,
DECL_ARGUMENTS (new_decl) = DECL_ARGUMENTS (old_decl);
initialize_cfun (new_decl, old_decl,
new_entry ? new_entry->count : old_entry_block->count);
+ new_version_node->calls_declare_variant_alt
+ = old_version_node->calls_declare_variant_alt;
if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
= id.src_cfun->gimple_df->ipa_pta;
@@ -2038,12 +2038,28 @@ execute_omp_device_lower ()
bool regimplify = false;
basic_block bb;
gimple_stmt_iterator gsi;
+ bool calls_declare_variant_alt
+ = cgraph_node::get (cfun->decl)->calls_declare_variant_alt;
FOR_EACH_BB_FN (bb, cfun)
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
- if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
+ if (!is_gimple_call (stmt))
continue;
+ if (!gimple_call_internal_p (stmt))
+ {
+ if (calls_declare_variant_alt)
+ if (tree fndecl = gimple_call_fndecl (stmt))
+ {
+ tree new_fndecl = omp_resolve_declare_variant (fndecl);
+ if (new_fndecl != fndecl)
+ {
+ gimple_call_set_fndecl (stmt, new_fndecl);
+ update_stmt (stmt);
+ }
+ }
+ continue;
+ }
tree lhs = gimple_call_lhs (stmt), rhs = NULL_TREE;
tree type = lhs ? TREE_TYPE (lhs) : integer_type_node;
switch (gimple_call_internal_fn (stmt))
@@ -2137,7 +2153,9 @@ public:
/* opt_pass methods: */
virtual bool gate (function *fun)
{
- return !(fun->curr_properties & PROP_gimple_lomp_dev);
+ return (!(fun->curr_properties & PROP_gimple_lomp_dev)
+ || (flag_openmp
+ && cgraph_node::get (fun->decl)->calls_declare_variant_alt));
}
virtual unsigned int execute (function *)
{
@@ -642,6 +642,8 @@ omp_maybe_offloaded (void)
if (symtab->state == PARSING)
/* Maybe. */
return true;
+ if (cfun && cfun->after_inlining)
+ return false;
if (current_function_decl
&& lookup_attribute ("omp declare target",
DECL_ATTRIBUTES (current_function_decl)))
@@ -694,8 +696,7 @@ omp_context_selector_matches (tree ctx)
(so in most of the cases), and we'd need to maintain set of
surrounding OpenMP constructs, which is better handled during
gimplification. */
- if (symtab->state == PARSING
- || (cfun->curr_properties & PROP_gimple_any) != 0)
+ if (symtab->state == PARSING)
{
ret = -1;
continue;
@@ -704,6 +705,28 @@ omp_context_selector_matches (tree ctx)
enum tree_code constructs[5];
int nconstructs
= omp_constructor_traits_to_codes (TREE_VALUE (t1), constructs);
+
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ {
+ if (!cfun->after_inlining)
+ {
+ ret = -1;
+ continue;
+ }
+ int i;
+ for (i = 0; i < nconstructs; ++i)
+ if (constructs[i] == OMP_SIMD)
+ break;
+ if (i < nconstructs)
+ {
+ ret = -1;
+ continue;
+ }
+ /* If there is no simd, assume it is ok after IPA,
+ constructs should have been checked before. */
+ continue;
+ }
+
int r = omp_construct_selector_matches (constructs, nconstructs,
NULL);
if (r == 0)
@@ -738,6 +761,9 @@ omp_context_selector_matches (tree ctx)
case 'a':
if (set == 'i' && !strcmp (sel, "atomic_default_mem_order"))
{
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ break;
+
enum omp_memory_order omo
= ((enum omp_memory_order)
(omp_requires_mask
@@ -816,6 +842,9 @@ omp_context_selector_matches (tree ctx)
case 'u':
if (set == 'i' && !strcmp (sel, "unified_address"))
{
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ break;
+
if ((omp_requires_mask & OMP_REQUIRES_UNIFIED_ADDRESS) == 0)
{
if (symtab->state == PARSING)
@@ -827,6 +856,9 @@ omp_context_selector_matches (tree ctx)
}
if (set == 'i' && !strcmp (sel, "unified_shared_memory"))
{
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ break;
+
if ((omp_requires_mask
& OMP_REQUIRES_UNIFIED_SHARED_MEMORY) == 0)
{
@@ -841,6 +873,9 @@ omp_context_selector_matches (tree ctx)
case 'd':
if (set == 'i' && !strcmp (sel, "dynamic_allocators"))
{
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ break;
+
if ((omp_requires_mask
& OMP_REQUIRES_DYNAMIC_ALLOCATORS) == 0)
{
@@ -855,6 +890,9 @@ omp_context_selector_matches (tree ctx)
case 'r':
if (set == 'i' && !strcmp (sel, "reverse_offload"))
{
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ break;
+
if ((omp_requires_mask & OMP_REQUIRES_REVERSE_OFFLOAD) == 0)
{
if (symtab->state == PARSING)
@@ -944,7 +982,8 @@ omp_context_selector_matches (tree ctx)
#pragma omp declare simd on it, some simd clones
might have the isa added later on. */
if (r == -1
- && targetm.simd_clone.compute_vecsize_and_simdlen)
+ && targetm.simd_clone.compute_vecsize_and_simdlen
+ && (cfun == NULL || !cfun->after_inlining))
{
tree attrs
= DECL_ATTRIBUTES (current_function_decl);
@@ -1415,6 +1454,191 @@ omp_context_compute_score (tree ctx, wid
return ret;
}
+/* Class describing a single variant. */
+struct GTY(()) omp_declare_variant_entry {
+ /* NODE of the variant. */
+ cgraph_node *variant;
+ /* Score if not in declare simd clone. */
+ widest_int score;
+ /* Score if in declare simd clone. */
+ widest_int score_in_declare_simd_clone;
+ /* Context selector for the variant. */
+ tree ctx;
+ /* True if the context selector is known to match already. */
+ bool matches;
+};
+
+/* Class describing a function with variants. */
+struct GTY((for_user)) omp_declare_variant_base_entry {
+ /* NODE of the base function. */
+ cgraph_node *base;
+ /* NODE of the artificial function created for the deferred variant
+ resolution. */
+ cgraph_node *node;
+ /* Vector of the variants. */
+ vec<omp_declare_variant_entry, va_gc> *variants;
+};
+
+struct omp_declare_variant_hasher
+ : ggc_ptr_hash<omp_declare_variant_base_entry> {
+ static hashval_t hash (omp_declare_variant_base_entry *);
+ static bool equal (omp_declare_variant_base_entry *,
+ omp_declare_variant_base_entry *);
+};
+
+hashval_t
+omp_declare_variant_hasher::hash (omp_declare_variant_base_entry *x)
+{
+ inchash::hash hstate;
+ hstate.add_int (DECL_UID (x->base->decl));
+ hstate.add_int (x->variants->length ());
+ omp_declare_variant_entry *variant;
+ unsigned int i;
+ FOR_EACH_VEC_SAFE_ELT (x->variants, i, variant)
+ {
+ hstate.add_int (DECL_UID (variant->variant->decl));
+ hstate.add_wide_int (variant->score);
+ hstate.add_wide_int (variant->score_in_declare_simd_clone);
+ hstate.add_ptr (variant->ctx);
+ hstate.add_int (variant->matches);
+ }
+ return hstate.end ();
+}
+
+bool
+omp_declare_variant_hasher::equal (omp_declare_variant_base_entry *x,
+ omp_declare_variant_base_entry *y)
+{
+ if (x->base != y->base
+ || x->variants->length () != y->variants->length ())
+ return false;
+ omp_declare_variant_entry *variant;
+ unsigned int i;
+ FOR_EACH_VEC_SAFE_ELT (x->variants, i, variant)
+ if (variant->variant != (*y->variants)[i].variant
+ || variant->score != (*y->variants)[i].score
+ || (variant->score_in_declare_simd_clone
+ != (*y->variants)[i].score_in_declare_simd_clone)
+ || variant->ctx != (*y->variants)[i].ctx
+ || variant->matches != (*y->variants)[i].matches)
+ return false;
+ return true;
+}
+
+static GTY(()) hash_table<omp_declare_variant_hasher> *omp_declare_variants;
+
+struct omp_declare_variant_alt_hasher
+ : ggc_ptr_hash<omp_declare_variant_base_entry> {
+ static hashval_t hash (omp_declare_variant_base_entry *);
+ static bool equal (omp_declare_variant_base_entry *,
+ omp_declare_variant_base_entry *);
+};
+
+hashval_t
+omp_declare_variant_alt_hasher::hash (omp_declare_variant_base_entry *x)
+{
+ return DECL_UID (x->node->decl);
+}
+
+bool
+omp_declare_variant_alt_hasher::equal (omp_declare_variant_base_entry *x,
+ omp_declare_variant_base_entry *y)
+{
+ return x->node == y->node;
+}
+
+static GTY(()) hash_table<omp_declare_variant_alt_hasher>
+ *omp_declare_variant_alt;
+
+/* Try to resolve declare variant after gimplification. */
+
+static tree
+omp_resolve_late_declare_variant (tree alt)
+{
+ cgraph_node *node = cgraph_node::get (alt);
+ cgraph_node *cur_node = cgraph_node::get (cfun->decl);
+ if (node == NULL
+ || !node->declare_variant_alt
+ || !cfun->after_inlining)
+ return alt;
+
+ omp_declare_variant_base_entry entry;
+ entry.base = NULL;
+ entry.node = node;
+ entry.variants = NULL;
+ omp_declare_variant_base_entry *entryp
+ = omp_declare_variant_alt->find_with_hash (&entry, DECL_UID (alt));
+
+ unsigned int i, j;
+ omp_declare_variant_entry *varentry1, *varentry2;
+ auto_vec <bool, 16> matches;
+ unsigned int nmatches = 0;
+ FOR_EACH_VEC_SAFE_ELT (entryp->variants, i, varentry1)
+ {
+ if (varentry1->matches)
+ {
+ /* This has been checked to be ok already. */
+ matches.safe_push (true);
+ nmatches++;
+ continue;
+ }
+ switch (omp_context_selector_matches (varentry1->ctx))
+ {
+ case 0:
+ matches.safe_push (false);
+ break;
+ case -1:
+ return alt;
+ default:
+ matches.safe_push (true);
+ nmatches++;
+ break;
+ }
+ }
+
+ if (nmatches == 0)
+ return entryp->base->decl;
+
+ /* A context selector that is a strict subset of another context selector
+ has a score of zero. */
+ FOR_EACH_VEC_SAFE_ELT (entryp->variants, i, varentry1)
+ if (matches[i])
+ {
+ for (j = i + 1;
+ vec_safe_iterate (entryp->variants, j, &varentry2); ++j)
+ if (matches[j])
+ {
+ int r = omp_context_selector_compare (varentry1->ctx,
+ varentry2->ctx);
+ if (r == -1)
+ {
+ /* ctx1 is a strict subset of ctx2, ignore ctx1. */
+ matches[i] = false;
+ break;
+ }
+ else if (r == 1)
+ /* ctx2 is a strict subset of ctx1, remove ctx2. */
+ matches[j] = false;
+ }
+ }
+
+ widest_int max_score = -1;
+ varentry2 = NULL;
+ FOR_EACH_VEC_SAFE_ELT (entryp->variants, i, varentry1)
+ if (matches[i])
+ {
+ widest_int score
+ = (cur_node->simdclone ? varentry1->score_in_declare_simd_clone
+ : varentry1->score);
+ if (score > max_score)
+ {
+ max_score = score;
+ varentry2 = varentry1;
+ }
+ }
+ return varentry2->variant->decl;
+}
+
/* Try to resolve declare variant, return the variant decl if it should
be used instead of base, or base otherwise. */
@@ -1422,6 +1646,9 @@ tree
omp_resolve_declare_variant (tree base)
{
tree variant1 = NULL_TREE, variant2 = NULL_TREE;
+ if (cfun && (cfun->curr_properties & PROP_gimple_any) != 0)
+ return omp_resolve_late_declare_variant (base);
+
auto_vec <tree, 16> variants;
auto_vec <bool, 16> defer;
bool any_deferred = false;
@@ -1459,6 +1686,10 @@ omp_resolve_declare_variant (tree base)
bool first = true;
unsigned int i;
tree attr1, attr2;
+ omp_declare_variant_base_entry entry;
+ entry.base = cgraph_node::get_create (base);
+ entry.node = NULL;
+ vec_alloc (entry.variants, variants.length ());
FOR_EACH_VEC_ELT (variants, i, attr1)
{
widest_int score1;
@@ -1498,6 +1729,14 @@ omp_resolve_declare_variant (tree base)
variant2 = defer[i] ? NULL_TREE : attr1;
}
}
+ omp_declare_variant_entry varentry;
+ varentry.variant
+ = cgraph_node::get_create (TREE_PURPOSE (TREE_VALUE (attr1)));
+ varentry.score = score1;
+ varentry.score_in_declare_simd_clone = score2;
+ varentry.ctx = ctx;
+ varentry.matches = !defer[i];
+ entry.variants->quick_push (varentry);
}
/* If there is a clear winner variant with the score which is not
@@ -1522,17 +1761,67 @@ omp_resolve_declare_variant (tree base)
}
}
if (variant1)
- return TREE_PURPOSE (TREE_VALUE (variant1));
+ {
+ vec_free (entry.variants);
+ return TREE_PURPOSE (TREE_VALUE (variant1));
+ }
+ }
+
+ if (omp_declare_variants == NULL)
+ omp_declare_variants
+ = hash_table<omp_declare_variant_hasher>::create_ggc (64);
+ omp_declare_variant_base_entry **slot
+ = omp_declare_variants->find_slot (&entry, INSERT);
+ if (*slot != NULL)
+ {
+ vec_free (entry.variants);
+ return (*slot)->node->decl;
}
- return base;
+ *slot = ggc_cleared_alloc<omp_declare_variant_base_entry> ();
+ (*slot)->base = entry.base;
+ (*slot)->node = entry.base;
+ (*slot)->variants = entry.variants;
+ tree alt = build_decl (DECL_SOURCE_LOCATION (base), FUNCTION_DECL,
+ DECL_NAME (base), TREE_TYPE (base));
+ DECL_ARTIFICIAL (alt) = 1;
+ DECL_IGNORED_P (alt) = 1;
+ TREE_STATIC (alt) = 1;
+ tree attributes = DECL_ATTRIBUTES (base);
+ if (lookup_attribute ("noipa", attributes) == NULL)
+ {
+ attributes = tree_cons (get_identifier ("noipa"), NULL, attributes);
+ if (lookup_attribute ("noinline", attributes) == NULL)
+ attributes = tree_cons (get_identifier ("noinline"), NULL,
+ attributes);
+ if (lookup_attribute ("noclone", attributes) == NULL)
+ attributes = tree_cons (get_identifier ("noclone"), NULL,
+ attributes);
+ if (lookup_attribute ("no_icf", attributes) == NULL)
+ attributes = tree_cons (get_identifier ("no_icf"), NULL,
+ attributes);
+ }
+ DECL_ATTRIBUTES (alt) = attributes;
+ DECL_INITIAL (alt) = error_mark_node;
+ (*slot)->node = cgraph_node::create (alt);
+ (*slot)->node->declare_variant_alt = 1;
+ (*slot)->node->create_reference (entry.base, IPA_REF_ADDR);
+ omp_declare_variant_entry *varentry;
+ FOR_EACH_VEC_SAFE_ELT (entry.variants, i, varentry)
+ (*slot)->node->create_reference (varentry->variant, IPA_REF_ADDR);
+ if (omp_declare_variant_alt == NULL)
+ omp_declare_variant_alt
+ = hash_table<omp_declare_variant_alt_hasher>::create_ggc (64);
+ *omp_declare_variant_alt->find_slot_with_hash (*slot, DECL_UID (alt),
+ INSERT) = *slot;
+ return alt;
}
if (variants.length () == 1)
return TREE_PURPOSE (TREE_VALUE (variants[0]));
- /* A context selector that is a strict subset of another context selector has a score
- of zero. */
+ /* A context selector that is a strict subset of another context selector
+ has a score of zero. */
tree attr1, attr2;
unsigned int i, j;
FOR_EACH_VEC_ELT (variants, i, attr1)
@@ -1948,3 +2237,5 @@ oacc_get_ifn_dim_arg (const gimple *stmt
gcc_checking_assert (axis >= 0 && axis < GOMP_DIM_MAX);
return (int) axis;
}
+
+#include "gt-omp-general.h"
@@ -915,6 +915,8 @@ symbol_table::create_edge (cgraph_node *
caller->decl);
else
edge->in_polymorphic_cdtor = caller->thunk.thunk_p;
+ if (callee)
+ caller->calls_declare_variant_alt |= callee->declare_variant_alt;
if (callee && symtab->state != LTO_STREAMING
&& edge->callee->comdat_local_p ())
@@ -0,0 +1,28 @@
+/* { dg-do compile { target vect_simd_clones } } */
+/* { dg-additional-options "-fdump-tree-gimple -fdump-tree-optimized" } */
+/* { dg-additional-options "-mno-sse3" { target { i?86-*-* x86_64-*-* } } } */
+
+int f01 (int);
+int f02 (int);
+int f03 (int);
+#pragma omp declare variant (f01) match (device={isa("avx512f")}) /* 4 or 8 */
+#pragma omp declare variant (f02) match (implementation={vendor(score(3):gnu)},device={kind(cpu)}) /* (1 or 2) + 3 */
+#pragma omp declare variant (f03) match (implementation={vendor(score(5):gnu)},device={kind(host)}) /* (1 or 2) + 5 */
+int f04 (int);
+
+#pragma omp declare simd
+int
+test1 (int x)
+{
+ /* At gimplification time, we can't decide yet which function to call. */
+ /* { dg-final { scan-tree-dump-times "f04 \\\(x" 2 "gimple" } } */
+ /* After simd clones are created, the original non-clone test1 shall
+ call f03 (score 6), the sse2/avx/avx2 clones too, but avx512f clones
+ shall call f01 with score 8. */
+ /* { dg-final { scan-tree-dump-not "f04 \\\(x" "optimized" } } */
+ /* { dg-final { scan-tree-dump-times "f03 \\\(x" 14 "optimized" } } */
+ /* { dg-final { scan-tree-dump-times "f01 \\\(x" 4 "optimized" } } */
+ int a = f04 (x);
+ int b = f04 (x);
+ return a + b;
+}