#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "tm.h"
#include "rtl.h"
#include "expr.h"
#include "hard-reg-set.h"
#include "basic-block.h"
#include "value-prof.h"
#include "output.h"
#include "flags.h"
#include "insn-config.h"
#include "recog.h"
#include "optabs.h"
#include "regs.h"
static struct value_prof_hooks *value_prof_hooks;
static void insn_divmod_values_to_profile (rtx, unsigned *,
struct histogram_value **);
static void insn_values_to_profile (rtx, unsigned *, struct histogram_value **);
static rtx gen_divmod_fixed_value (enum machine_mode, enum rtx_code, rtx, rtx,
rtx, gcov_type);
static rtx gen_mod_pow2 (enum machine_mode, enum rtx_code, rtx, rtx, rtx);
static rtx gen_mod_subtract (enum machine_mode, enum rtx_code, rtx, rtx, rtx,
int);
static bool divmod_fixed_value_transform (rtx insn);
static bool mod_pow2_value_transform (rtx);
static bool mod_subtract_transform (rtx);
void
free_profiled_values (unsigned n_values ATTRIBUTE_UNUSED,
struct histogram_value *values)
{
free (values);
}
static void
insn_divmod_values_to_profile (rtx insn, unsigned *n_values,
struct histogram_value **values)
{
rtx set, set_src, op1, op2;
enum machine_mode mode;
if (!INSN_P (insn))
return;
set = single_set (insn);
if (!set)
return;
mode = GET_MODE (SET_DEST (set));
if (!INTEGRAL_MODE_P (mode))
return;
set_src = SET_SRC (set);
switch (GET_CODE (set_src))
{
case DIV:
case MOD:
case UDIV:
case UMOD:
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
if (side_effects_p (op2))
return;
if ((GET_CODE (set_src) == UMOD) && !CONSTANT_P (op2))
{
*values = xrealloc (*values,
(*n_values + 1)
* sizeof (struct histogram_value));
(*values)[*n_values].value = op2;
(*values)[*n_values].seq = NULL_RTX;
(*values)[*n_values].mode = mode;
(*values)[*n_values].insn = insn;
(*values)[*n_values].type = HIST_TYPE_POW2;
(*values)[*n_values].hdata.pow2.may_be_other = 1;
(*n_values)++;
}
if (!CONSTANT_P (op2))
{
*values = xrealloc (*values,
(*n_values + 1)
* sizeof (struct histogram_value));
(*values)[*n_values].value = op2;
(*values)[*n_values].mode = mode;
(*values)[*n_values].seq = NULL_RTX;
(*values)[*n_values].insn = insn;
(*values)[*n_values].type = HIST_TYPE_SINGLE_VALUE;
(*n_values)++;
}
if (GET_CODE (set_src) == UMOD && !side_effects_p (op1))
{
rtx tmp;
*values = xrealloc (*values,
(*n_values + 1)
* sizeof (struct histogram_value));
start_sequence ();
tmp = simplify_gen_binary (DIV, mode, copy_rtx (op1), copy_rtx (op2));
(*values)[*n_values].value = force_operand (tmp, NULL_RTX);
(*values)[*n_values].seq = get_insns ();
end_sequence ();
(*values)[*n_values].mode = mode;
(*values)[*n_values].insn = insn;
(*values)[*n_values].type = HIST_TYPE_INTERVAL;
(*values)[*n_values].hdata.intvl.int_start = 0;
(*values)[*n_values].hdata.intvl.steps = 2;
(*values)[*n_values].hdata.intvl.may_be_less = 1;
(*values)[*n_values].hdata.intvl.may_be_more = 1;
(*n_values)++;
}
return;
default:
return;
}
}
static void
insn_values_to_profile (rtx insn,
unsigned *n_values,
struct histogram_value **values)
{
if (flag_value_profile_transformations)
insn_divmod_values_to_profile (insn, n_values, values);
}
static void
rtl_find_values_to_profile (unsigned *n_values, struct histogram_value **values)
{
rtx insn;
unsigned i;
life_analysis (NULL, PROP_DEATH_NOTES);
*n_values = 0;
*values = NULL;
for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
insn_values_to_profile (insn, n_values, values);
for (i = 0; i < *n_values; i++)
{
switch ((*values)[i].type)
{
case HIST_TYPE_INTERVAL:
if (dump_file)
fprintf (dump_file,
"Interval counter for insn %d, range %d -- %d.\n",
INSN_UID ((rtx)(*values)[i].insn),
(*values)[i].hdata.intvl.int_start,
((*values)[i].hdata.intvl.int_start
+ (*values)[i].hdata.intvl.steps - 1));
(*values)[i].n_counters = (*values)[i].hdata.intvl.steps +
((*values)[i].hdata.intvl.may_be_less ? 1 : 0) +
((*values)[i].hdata.intvl.may_be_more ? 1 : 0);
break;
case HIST_TYPE_POW2:
if (dump_file)
fprintf (dump_file,
"Pow2 counter for insn %d.\n",
INSN_UID ((rtx)(*values)[i].insn));
(*values)[i].n_counters
= GET_MODE_BITSIZE ((*values)[i].mode)
+ ((*values)[i].hdata.pow2.may_be_other ? 1 : 0);
break;
case HIST_TYPE_SINGLE_VALUE:
if (dump_file)
fprintf (dump_file,
"Single value counter for insn %d.\n",
INSN_UID ((rtx)(*values)[i].insn));
(*values)[i].n_counters = 3;
break;
case HIST_TYPE_CONST_DELTA:
if (dump_file)
fprintf (dump_file,
"Constant delta counter for insn %d.\n",
INSN_UID ((rtx)(*values)[i].insn));
(*values)[i].n_counters = 4;
break;
default:
abort ();
}
}
allocate_reg_info (max_reg_num (), FALSE, FALSE);
}
static bool
rtl_value_profile_transformations (void)
{
rtx insn, next;
int changed = false;
for (insn = get_insns (); insn; insn = next)
{
next = NEXT_INSN (insn);
if (!INSN_P (insn))
continue;
if (!find_reg_note (insn, REG_VALUE_PROFILE, 0))
continue;
if (!maybe_hot_bb_p (BLOCK_FOR_INSN (insn)))
continue;
if (dump_file)
{
fprintf (dump_file, "Trying transformations on insn %d\n",
INSN_UID (insn));
print_rtl_single (dump_file, insn);
}
if (flag_value_profile_transformations
&& (mod_subtract_transform (insn)
|| divmod_fixed_value_transform (insn)
|| mod_pow2_value_transform (insn)))
changed = true;
}
if (changed)
{
commit_edge_insertions ();
allocate_reg_info (max_reg_num (), FALSE, FALSE);
}
return changed;
}
static rtx
gen_divmod_fixed_value (enum machine_mode mode, enum rtx_code operation,
rtx target, rtx op1, rtx op2, gcov_type value)
{
rtx tmp, tmp1;
rtx neq_label = gen_label_rtx ();
rtx end_label = gen_label_rtx ();
rtx sequence;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
do_compare_rtx_and_jump (tmp, GEN_INT (value), NE, 0, mode, NULL_RTX,
NULL_RTX, neq_label);
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (op1), GEN_INT (value));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (copy_rtx (target), copy_rtx (tmp1));
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
emit_label (neq_label);
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (op1), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (copy_rtx (target), copy_rtx (tmp1));
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static bool
divmod_fixed_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type val, count, all;
edge e;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != DIV && code != MOD && code != UDIV && code != UMOD)
return false;
op1 = XEXP (set_src, false);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_SINGLE_VALUE))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
val = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
all = INTVAL (XEXP (histogram, 0));
if (!rtx_equal_p (op2, value) || 2 * count < all)
return false;
if (dump_file)
fprintf (dump_file, "Div/mod by constant transformation on insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_divmod_fixed_value (mode, code, set_dest, op1, op2, val), e);
return true;
}
static rtx
gen_mod_pow2 (enum machine_mode mode, enum rtx_code operation, rtx target,
rtx op1, rtx op2)
{
rtx tmp, tmp1, tmp2, tmp3;
rtx neq_label = gen_label_rtx ();
rtx end_label = gen_label_rtx ();
rtx sequence;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
tmp1 = expand_simple_binop (mode, PLUS, tmp, constm1_rtx, NULL_RTX,
0, OPTAB_WIDEN);
tmp2 = expand_simple_binop (mode, AND, tmp, tmp1, NULL_RTX,
0, OPTAB_WIDEN);
do_compare_rtx_and_jump (tmp2, const0_rtx, NE, 0, mode, NULL_RTX,
NULL_RTX, neq_label);
tmp3 = expand_simple_binop (mode, AND, op1, tmp1, target,
0, OPTAB_WIDEN);
if (tmp3 != target)
emit_move_insn (copy_rtx (target), tmp3);
emit_jump_insn (gen_jump (end_label));
emit_barrier ();
emit_label (neq_label);
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (op1), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (target, tmp1);
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static bool
mod_pow2_value_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, count;
edge e;
int i;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_POW2))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
wrong_values =INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
count = 0;
for (i = 0; i < GET_MODE_BITSIZE (mode); i++)
{
count += INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
}
if (!rtx_equal_p (op2, value))
return false;
if (count < wrong_values)
return false;
if (dump_file)
fprintf (dump_file, "Mod power of 2 transformation on insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_pow2 (mode, code, set_dest, op1, op2), e);
return true;
}
static rtx
gen_mod_subtract (enum machine_mode mode, enum rtx_code operation,
rtx target, rtx op1, rtx op2, int sub)
{
rtx tmp, tmp1;
rtx end_label = gen_label_rtx ();
rtx sequence;
int i;
start_sequence ();
if (!REG_P (op2))
{
tmp = gen_reg_rtx (mode);
emit_move_insn (tmp, copy_rtx (op2));
}
else
tmp = op2;
emit_move_insn (target, copy_rtx (op1));
do_compare_rtx_and_jump (target, tmp, LTU, 0, mode, NULL_RTX,
NULL_RTX, end_label);
for (i = 0; i < sub; i++)
{
tmp1 = expand_simple_binop (mode, MINUS, target, tmp, target,
0, OPTAB_WIDEN);
if (tmp1 != target)
emit_move_insn (target, tmp1);
do_compare_rtx_and_jump (target, tmp, LTU, 0, mode, NULL_RTX,
NULL_RTX, end_label);
}
tmp1 = simplify_gen_binary (operation, mode, copy_rtx (target), copy_rtx (tmp));
tmp1 = force_operand (tmp1, target);
if (tmp1 != target)
emit_move_insn (target, tmp1);
emit_label (end_label);
sequence = get_insns ();
end_sequence ();
rebuild_jump_labels (sequence);
return sequence;
}
static bool
mod_subtract_transform (rtx insn)
{
rtx set, set_src, set_dest, op1, op2, value, histogram;
enum rtx_code code;
enum machine_mode mode;
gcov_type wrong_values, counts[2], count, all;
edge e;
int i;
set = single_set (insn);
if (!set)
return false;
set_src = SET_SRC (set);
set_dest = SET_DEST (set);
code = GET_CODE (set_src);
mode = GET_MODE (set_dest);
if (code != UMOD)
return false;
op1 = XEXP (set_src, 0);
op2 = XEXP (set_src, 1);
for (histogram = REG_NOTES (insn);
histogram;
histogram = XEXP (histogram, 1))
if (REG_NOTE_KIND (histogram) == REG_VALUE_PROFILE
&& XEXP (XEXP (histogram, 0), 0) == GEN_INT (HIST_TYPE_INTERVAL))
break;
if (!histogram)
return false;
histogram = XEXP (XEXP (histogram, 0), 1);
value = XEXP (histogram, 0);
histogram = XEXP (histogram, 1);
all = 0;
for (i = 0; i < 2; i++)
{
counts[i] = INTVAL (XEXP (histogram, 0));
all += counts[i];
histogram = XEXP (histogram, 1);
}
wrong_values = INTVAL (XEXP (histogram, 0));
histogram = XEXP (histogram, 1);
wrong_values += INTVAL (XEXP (histogram, 0));
all += wrong_values;
count = 0;
for (i = 0; i < 2; i++)
{
count += counts[i];
if (count * 2 >= all)
break;
}
if (i == 2)
return false;
if (dump_file)
fprintf (dump_file, "Mod subtract transformation on insn %d\n",
INSN_UID (insn));
e = split_block (BLOCK_FOR_INSN (insn), PREV_INSN (insn));
delete_insn (insn);
insert_insn_on_edge (
gen_mod_subtract (mode, code, set_dest, op1, op2, i), e);
return true;
}
struct value_prof_hooks {
void (*find_values_to_profile) (unsigned *, struct histogram_value **);
bool (*value_profile_transformations) (void);
};
static struct value_prof_hooks rtl_value_prof_hooks =
{
rtl_find_values_to_profile,
rtl_value_profile_transformations
};
void
rtl_register_value_prof_hooks (void)
{
value_prof_hooks = &rtl_value_prof_hooks;
if (ir_type ())
abort ();
}
static void
tree_find_values_to_profile (unsigned *n_values, struct histogram_value **values)
{
(void)n_values;
(void)values;
abort ();
}
static bool
tree_value_profile_transformations (void)
{
abort ();
}
static struct value_prof_hooks tree_value_prof_hooks = {
tree_find_values_to_profile,
tree_value_profile_transformations
};
void
tree_register_value_prof_hooks (void)
{
value_prof_hooks = &tree_value_prof_hooks;
if (!ir_type ())
abort ();
}
void
find_values_to_profile (unsigned *n_values, struct histogram_value **values)
{
(value_prof_hooks->find_values_to_profile) (n_values, values);
}
bool
value_profile_transformations (void)
{
return (value_prof_hooks->value_profile_transformations) ();
}