gcc - GNU Compiler Collection (original) (raw)

This patch makes the AFDO's VPT to happen during early inlining. This should make the einline pass inside afdo pass unnecesary, but some inlining still happens there - I will need to debug why that happens and will try to drop the afdo's inliner incrementally. get_inline_stack_in_node can now be used to produce inline stack out of callgraph nodes which are marked as inline clones, so we do not need to iterate tree-inline and IPA decisions phases like old code did. I also added some debug facilities - dumping of decisions and inline stacks, so one can match them with data in gcov profile. Former VPT pass identified all caes where in train run indirect call was inlined and the inlined callee collected some samples. In this case it forced inline without doing any checks, such as whether inlining is possible. New code simply introduces speculative edges into callgraph and lets afdo inlining to decide. Old code also marked statements that were introduced during promotion to prevent doing double speculation i.e. if (ptr == foo) .. inlined foo ... else ptr (); to if (ptr == foo) .. inlined foo ... else if (ptr == foo) foo (); // for IPA inlining else ptr (); Since inlning now happens much earlier, tracking the statements would be quite hard. Instead I simply remove the targets from profile data which sould have same effect. I also noticed that there is nothing setting max_count so all non-0 profile is considered hot which I fixed too. Training with ref run I now get 500.perlbench_r 1 160 9.93 * 1 162 9.84 * 502.gcc_r NR NR 505.mcf_r 1 186 8.68 * 1 194 8.34 * 520.omnetpp_r 1 183 7.15 * 1 208 6.32 * 523.xalancbmk_r NR NR 525.x264_r 1 85.2 20.5 * 1 85.8 20.4 * 531.deepsjeng_r 1 165 6.93 * 1 176 6.51 * 541.leela_r 1 268 6.18 * 1 282 5.87 * 548.exchange2_r 1 86.3 30.4 * 1 88.9 29.5 * 557.xz_r 1 224 4.81 * 1 224 4.82 * Est. SPECrate2017_int_base 9.72 Est. SPECrate2017_int_peak 9.33 503.bwaves_r NR NR 507.cactuBSSN_r 1 107 11.9 * 1 105 12.0 * 508.namd_r 1 108 8.79 * 1 116 8.18 * 510.parest_r 1 143 18.3 * 1 156 16.8 * 511.povray_r 1 188 12.4 * 1 163 14.4 * 519.lbm_r 1 72.0 14.6 * 1 75.0 14.1 * 521.wrf_r 1 106 21.1 * 1 106 21.1 * 526.blender_r 1 147 10.3 * 1 147 10.4 * 527.cam4_r 1 110 15.9 * 1 118 14.8 * 538.imagick_r 1 104 23.8 * 1 105 23.7 * 544.nab_r 1 146 11.6 * 1 143 11.8 * 549.fotonik3d_r 1 134 29.0 * 1 169 23.1 * 554.roms_r 1 86.6 18.4 * 1 89.3 17.8 * Est. SPECrate2017_fp_base 15.4 Est. SPECrate2017_fp_peak 14.9 Base is without profile feedback and peak is AFDO. gcc/ChangeLog: * auto-profile.cc (dump_inline_stack): New function. (get_inline_stack_in_node): New function. (get_relative_location_for_stmt): Add FN parameter. (has_indirect_call): Remove. (function_instance::find_icall_target_map): Add FN parameter. (function_instance::remove_icall_target): New function. (function_instance::read_function_instance): Set sum_max. (autofdo_source_profile::get_count_info): Add NODE parameter. (autofdo_source_profile::update_inlined_ind_target): Add NODE parameter. (autofdo_source_profile::remove_icall_target): New function. (afdo_indirect_call): Add INDIRECT_EDGE parameter; dump reason for failure; do not check for recursion; do not inline call. (afdo_vpt): Add INDIRECT_EDGE parameter. (afdo_set_bb_count): Do not take PROMOTED set. (afdo_vpt_for_early_inline): Remove. (afdo_annotate_cfg): Do not take PROMOTED set. (auto_profile): Do not call afdo_vpt_for_early_inline. (afdo_callsite_hot_enough_for_early_inline): Dump count. (remove_afdo_speculative_target): New function. * auto-profile.h (afdo_vpt_for_early_inline): Declare. (remove_afdo_speculative_target): Declare. * ipa-inline.cc (inline_functions_by_afdo): Do VPT. (early_inliner): Redirecct edges if inlining happened. * tree-inline.cc (expand_call_inline): Add sanity check. gcc/testsuite/ChangeLog: * gcc.dg/tree-prof/afdo-vpt-earlyinline.c: Update template. * gcc.dg/tree-prof/indir-call-prof-2.c: Update template.

@@ -243,9 +243,13 @@ public:

243

is found. */

243

is found. */

244

bool get_count_info (location_t loc, count_info *info) const;

244

bool get_count_info (location_t loc, count_info *info) const;

245

245

246

/* Read the inlined indirect call target profile for STMT and store it in

246

/* Read the inlined indirect call target profile for STMT in FN and store it

247

MAP, return the total count for all inlined indirect calls. */

247

in MAP, return the total count for all inlined indirect calls. */

248

gcov_type find_icall_target_map (gcall *stmt, icall_target_map *map) const;

248

gcov_type find_icall_target_map (tree fn, gcall *stmt,

249

icall_target_map *map) const;

250

251

/* Remove inlined indirect call target profile for STMT in FN. */

252

void remove_icall_target (tree fn, gcall *stmt);

249

253

250

/* Mark LOC as annotated. */

254

/* Mark LOC as annotated. */

251

void mark_annotated (location_t loc);

255

void mark_annotated (location_t loc);

@@ -302,20 +306,26 @@ public:

302

function_instance *get_function_instance_by_decl (tree decl) const;

306

function_instance *get_function_instance_by_decl (tree decl) const;

303

307

304

/* Find count_info for a given gimple STMT. If found, store the count_info

308

/* Find count_info for a given gimple STMT. If found, store the count_info

305

in INFO and return true; otherwise return false. */

309

in INFO and return true; otherwise return false.

306

bool get_count_info (gimple *stmt, count_info *info) const;

310

NODE can be used to specify particular inline clone. */

311

bool get_count_info (gimple *stmt, count_info *info,

312

cgraph_node *node = NULL) const;

307

313

308

/* Find count_info for a given gimple location GIMPLE_LOC. If found,

314

/* Find count_info for a given gimple location GIMPLE_LOC. If found,

309

store the count_info in INFO and return true; otherwise return false. */

315

store the count_info in INFO and return true; otherwise return false.

310

bool get_count_info (location_t gimple_loc, count_info *info) const;

316

NODE can be used to specify particular inline clone. */

317

bool get_count_info (location_t gimple_loc, count_info *info,

318

cgraph_node *node = NULL) const;

311

319

312

/* Find total count of the callee of EDGE. */

320

/* Find total count of the callee of EDGE. */

313

gcov_type get_callsite_total_count (struct cgraph_edge *edge) const;

321

gcov_type get_callsite_total_count (struct cgraph_edge *edge) const;

314

322

315

/* Update value profile INFO for STMT from the inlined indirect callsite.

323

/* Update value profile INFO for STMT within NODE from the inlined indirect

316

Return true if INFO is updated. */

324

callsite. Return true if INFO is updated. */

317

bool update_inlined_ind_target (gcall *stmt, count_info *info);

325

bool update_inlined_ind_target (gcall *stmt, count_info *info,

326

cgraph_node *node);

318

327

328

void remove_icall_target (cgraph_edge *e);

319

private:

329

private:

320

/* Map from function_instance name index (in string_table) to

330

/* Map from function_instance name index (in string_table) to

321

function_instance. */

331

function_instance. */

@@ -383,6 +393,24 @@ get_function_decl_from_block (tree block)

383

return BLOCK_ABSTRACT_ORIGIN (block);

393

return BLOCK_ABSTRACT_ORIGIN (block);

384

}

394

}

385

395

396

/* Dump STACK to F. */

397

398

static void

399

dump_inline_stack (FILE *f, inline_stack *stack)

400

{

401

bool first = true;

402

for (decl_lineno &p : *stack)

403

{

404

fprintf(f, "%s%s:%i.%i",

405

first ? "" : "; ",

406

IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (p.first)),

407

p.second >> 16,

408

p.second & 65535);

409

first = false;

410

}

411

fprintf (f, "\n");

412

}

413

386

/* Store inline stack for STMT in STACK. */

414

/* Store inline stack for STMT in STACK. */

387

415

388

static void

416

static void

@@ -412,12 +440,35 @@ get_inline_stack (location_t locus, inline_stack *stack,

412

stack->safe_push (std::make_pair (fn, get_combined_location (locus, fn)));

440

stack->safe_push (std::make_pair (fn, get_combined_location (locus, fn)));

413

}

441

}

414

442

443

/* Same as get_inline_stack for a given node which may be

444

an inline clone. If NODE is NULL, assume current_function_decl. */

445

static void

446

get_inline_stack_in_node (location_t locus, inline_stack *stack,

447

cgraph_node *node)

448

{

449

if (!node)

450

return get_inline_stack (locus, stack);

451

do

452

{

453

get_inline_stack (locus, stack, node->decl);

454

/* If caller is inlined, continue building stack. */

455

if (!node->inlined_to)

456

node = NULL;

457

else

458

{

459

locus = gimple_location (node->callers->call_stmt);

460

node = node->callers->caller;

461

}

462

}

463

while (node);

464

}

465

415

/* Return STMT's combined location, which is a 32bit integer in which

466

/* Return STMT's combined location, which is a 32bit integer in which

416

higher 16 bits stores the line offset of LOC to the start lineno

467

higher 16 bits stores the line offset of LOC to the start lineno

417

of DECL, The lower 16 bits stores the discriminator. */

468

of DECL, The lower 16 bits stores the discriminator. */

418

469

419

static unsigned

470

static unsigned

420

get_relative_location_for_stmt (gimple *stmt)

471

get_relative_location_for_stmt (tree fn, gimple *stmt)

421

{

472

{

422

location_t locus = gimple_location (stmt);

473

location_t locus = gimple_location (stmt);

423

if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)

474

if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)

@@ -428,25 +479,7 @@ get_relative_location_for_stmt (gimple *stmt)

428

if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block)) != UNKNOWN_LOCATION)

479

if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block)) != UNKNOWN_LOCATION)

429

return get_combined_location (locus,

480

return get_combined_location (locus,

430

get_function_decl_from_block (block));

481

get_function_decl_from_block (block));

431

return get_combined_location (locus, current_function_decl);

482

return get_combined_location (locus, fn);

432

}

433

434

/* Return true if BB contains indirect call. */

435

436

static bool

437

has_indirect_call (basic_block bb)

438

{

439

gimple_stmt_iterator gsi;

440

441

for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))

442

{

443

gimple *stmt = gsi_stmt (gsi);

444

if (gimple_code (stmt) == GIMPLE_CALL && !gimple_call_internal_p (stmt)

445

&& (gimple_call_fn (stmt) == NULL

446

|| TREE_CODE (gimple_call_fn (stmt)) != FUNCTION_DECL))

447

return true;

448

}

449

return false;

450

}

483

}

451

484

452

/* Member functions for string_table. */

485

/* Member functions for string_table. */

@@ -584,8 +617,7 @@ void function_instance::merge (function_instance *other)

584

pos_counts[iter->first].count += iter->second.count;

617

pos_counts[iter->first].count += iter->second.count;

585

}

618

}

586

619

587

/* Store the profile info for LOC in INFO. Return TRUE if profile info

620

/* Return profile info for LOC in INFO. */

588

is found. */

589

621

590

bool

622

bool

591

function_instance::get_count_info (location_t loc, count_info *info) const

623

function_instance::get_count_info (location_t loc, count_info *info) const

@@ -601,11 +633,11 @@ function_instance::get_count_info (location_t loc, count_info *info) const

601

MAP, return the total count for all inlined indirect calls. */

633

MAP, return the total count for all inlined indirect calls. */

602

634

603

gcov_type

635

gcov_type

604

function_instance::find_icall_target_map (gcall *stmt,

636

function_instance::find_icall_target_map (tree fn, gcall *stmt,

605

icall_target_map *map) const

637

icall_target_map *map) const

606

{

638

{

607

gcov_type ret = 0;

639

gcov_type ret = 0;

608

unsigned stmt_offset = get_relative_location_for_stmt (stmt);

640

unsigned stmt_offset = get_relative_location_for_stmt (fn, stmt);

609

641

610

for (callsite_map::const_iterator iter = callsites.begin ();

642

for (callsite_map::const_iterator iter = callsites.begin ();

611

iter != callsites.end (); ++iter)

643

iter != callsites.end (); ++iter)

@@ -624,6 +656,28 @@ function_instance::find_icall_target_map (gcall *stmt,

624

return ret;

656

return ret;

625

}

657

}

626

658

659

/* Remove the inlined indirect call target profile for STMT. */

660

661

void

662

function_instance::remove_icall_target (tree fn, gcall *stmt)

663

{

664

unsigned stmt_offset = get_relative_location_for_stmt (fn, stmt);

665

int n = 0;

666

667

for (callsite_map::const_iterator iter = callsites.begin ();

668

iter != callsites.end ();)

669

if (iter->first.first != stmt_offset)

670

++iter;

671

else

672

{

673

iter = callsites.erase (iter);

674

n++;

675

}

676

/* TODO: If we add support for multiple targets, we may want to

677

remove only those we succesfully inlined. */

678

gcc_assert (n);

679

}

680

627

/* Read the profile and create a function_instance with head count as

681

/* Read the profile and create a function_instance with head count as

628

HEAD_COUNT. Recursively read callsites to create nested function_instances

682

HEAD_COUNT. Recursively read callsites to create nested function_instances

629

too. STACK is used to track the recursive creation process. */

683

too. STACK is used to track the recursive creation process. */

@@ -671,6 +725,9 @@ function_instance::read_function_instance (function_instance_stack *stack,

671

unsigned num_targets = gcov_read_unsigned ();

725

unsigned num_targets = gcov_read_unsigned ();

672

gcov_type count = gcov_read_counter ();

726

gcov_type count = gcov_read_counter ();

673

s->pos_counts[offset].count = count;

727

s->pos_counts[offset].count = count;

728

afdo_profile_info->sum_max = std::max (afdo_profile_info->sum_max,

729

count);

730

674

for (unsigned j = 0; j < stack->length (); j++)

731

for (unsigned j = 0; j < stack->length (); j++)

675

(*stack)[j]->total_count_ += count;

732

(*stack)[j]->total_count_ += count;

676

for (unsigned j = 0; j < num_targets; j++)

733

for (unsigned j = 0; j < num_targets; j++)

@@ -718,20 +775,22 @@ autofdo_source_profile::get_function_instance_by_decl (tree decl) const

718

in INFO and return true; otherwise return false. */

775

in INFO and return true; otherwise return false. */

719

776

720

bool

777

bool

721

autofdo_source_profile::get_count_info (gimple *stmt, count_info *info) const

778

autofdo_source_profile::get_count_info (gimple *stmt, count_info *info,

779

cgraph_node *node) const

722

{

780

{

723

return get_count_info (gimple_location (stmt), info);

781

return get_count_info (gimple_location (stmt), info, node);

724

}

782

}

725

783

726

bool

784

bool

727

autofdo_source_profile::get_count_info (location_t gimple_loc,

785

autofdo_source_profile::get_count_info (location_t gimple_loc,

728

count_info *info) const

786

count_info *info,

787

cgraph_node *node) const

729

{

788

{

730

if (LOCATION_LOCUS (gimple_loc) == cfun->function_end_locus)

789

if (LOCATION_LOCUS (gimple_loc) == cfun->function_end_locus)

731

return false;

790

return false;

732

791

733

inline_stack stack;

792

inline_stack stack;

734

get_inline_stack (gimple_loc, &stack);

793

get_inline_stack_in_node (gimple_loc, &stack, node);

735

if (stack.length () == 0)

794

if (stack.length () == 0)

736

return false;

795

return false;

737

function_instance *s = get_function_instance_by_inline_stack (stack);

796

function_instance *s = get_function_instance_by_inline_stack (stack);

@@ -745,7 +804,8 @@ autofdo_source_profile::get_count_info (location_t gimple_loc,

745

804

746

bool

805

bool

747

autofdo_source_profile::update_inlined_ind_target (gcall *stmt,

806

autofdo_source_profile::update_inlined_ind_target (gcall *stmt,

748

count_info *info)

807

count_info *info,

808

cgraph_node *node)

749

{

809

{

750

if (dump_file)

810

if (dump_file)

751

{

811

{

@@ -756,12 +816,12 @@ autofdo_source_profile::update_inlined_ind_target (gcall *stmt,

756

if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)

816

if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)

757

{

817

{

758

if (dump_file)

818

if (dump_file)

759

fprintf (dump_file, " good locus\n");

819

fprintf (dump_file, " bad locus (funciton end)\n");

760

return false;

820

return false;

761

}

821

}

762

822

763

count_info old_info;

823

count_info old_info;

764

get_count_info (stmt, &old_info);

824

get_count_info (stmt, &old_info, node);

765

gcov_type total = 0;

825

gcov_type total = 0;

766

for (icall_target_map::const_iterator iter = old_info.targets.begin ();

826

for (icall_target_map::const_iterator iter = old_info.targets.begin ();

767

iter != old_info.targets.end (); ++iter)

827

iter != old_info.targets.end (); ++iter)

@@ -784,7 +844,7 @@ autofdo_source_profile::update_inlined_ind_target (gcall *stmt,

784

}

844

}

785

845

786

inline_stack stack;

846

inline_stack stack;

787

get_inline_stack (gimple_location (stmt), &stack);

847

get_inline_stack_in_node (gimple_location (stmt), &stack, node);

788

if (stack.length () == 0)

848

if (stack.length () == 0)

789

{

849

{

790

if (dump_file)

850

if (dump_file)

@@ -795,24 +855,46 @@ autofdo_source_profile::update_inlined_ind_target (gcall *stmt,

795

if (s == NULL)

855

if (s == NULL)

796

{

856

{

797

if (dump_file)

857

if (dump_file)

798

fprintf (dump_file, " function not found in inline stack\n");

858

{

859

fprintf (dump_file, " function not found in inline stack:");

860

dump_inline_stack (dump_file, &stack);

861

}

799

return false;

862

return false;

800

}

863

}

801

icall_target_map map;

864

icall_target_map map;

802

if (s->find_icall_target_map (stmt, &map) == 0)

865

if (s->find_icall_target_map (node ? node->decl

866

: current_function_decl,

867

stmt, &map) == 0)

803

{

868

{

804

if (dump_file)

869

if (dump_file)

805

fprintf (dump_file, " no target map\n");

870

{

871

fprintf (dump_file, " no target map for stack: ");

872

dump_inline_stack (dump_file, &stack);

873

}

806

return false;

874

return false;

807

}

875

}

808

for (icall_target_map::const_iterator iter = map.begin ();

876

for (icall_target_map::const_iterator iter = map.begin ();

809

iter != map.end (); ++iter)

877

iter != map.end (); ++iter)

810

info->targets[iter->first] = iter->second;

878

info->targets[iter->first] = iter->second;

811

if (dump_file)

879

if (dump_file)

812

fprintf (dump_file, " looks good\n");

880

{

881

fprintf (dump_file, " looks good; stack:");

882

dump_inline_stack (dump_file, &stack);

883

}

813

return true;

884

return true;

814

}

885

}

815

886

887

void

888

autofdo_source_profile::remove_icall_target (cgraph_edge *e)

889

{

890

autofdo::inline_stack stack;

891

autofdo::get_inline_stack_in_node (gimple_location (e->call_stmt),

892

&stack, e->caller);

893

autofdo::function_instance *s

894

= get_function_instance_by_inline_stack (stack);

895

s->remove_icall_target (e->caller->decl, e->call_stmt);

896

}

897

816

/* Find total count of the callee of EDGE. */

898

/* Find total count of the callee of EDGE. */

817

899

818

gcov_type

900

gcov_type

@@ -822,18 +904,8 @@ autofdo_source_profile::get_callsite_total_count (

822

inline_stack stack;

904

inline_stack stack;

823

stack.safe_push (std::make_pair (edge->callee->decl, 0));

905

stack.safe_push (std::make_pair (edge->callee->decl, 0));

824

906

825

cgraph_edge *e = edge;

907

get_inline_stack_in_node (gimple_location (edge->call_stmt), &stack,

826

do

908

edge->caller);

827

{

828

get_inline_stack (gimple_location (e->call_stmt), &stack,

829

e->caller->decl);

830

/* If caller is inlined, continue building stack. */

831

if (!e->caller->inlined_to)

832

e = NULL;

833

else

834

e = e->caller->callers;

835

}

836

while (e);

837

909

838

function_instance *s = get_function_instance_by_inline_stack (stack);

910

function_instance *s = get_function_instance_by_inline_stack (stack);

839

if (s == NULL

911

if (s == NULL

@@ -1000,19 +1072,35 @@ read_profile (void)

1000

decide if it needs to promote and inline. */

1072

decide if it needs to promote and inline. */

1001

1073

1002

static bool

1074

static bool

1003

afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,

1075

afdo_indirect_call (gcall *stmt, const icall_target_map &map,

1004

bool transform)

1076

bool transform, cgraph_edge *indirect_edge)

1005

{

1077

{

1006

gimple *gs = gsi_stmt (*gsi);

1007

tree callee;

1078

tree callee;

1008

1079

1009

if (map.size () == 0)

1080

if (map.size () == 0)

1010

return false;

1081

{

1011

gcall *stmt = dyn_cast <gcall *> (gs);

1082

if (dump_file)

1012

if (!stmt

1083

fprintf (dump_file, "No targets found\n");

1013

|| gimple_call_internal_p (stmt)

1084

return false;

1014

|| gimple_call_fndecl (stmt) != NULL_TREE)

1085

}

1015

return false;

1086

if (!stmt)

1087

{

1088

if (dump_file)

1089

fprintf (dump_file, "No call statement\n");

1090

return false;

1091

}

1092

if (gimple_call_internal_p (stmt))

1093

{

1094

if (dump_file)

1095

fprintf (dump_file, "Internal call\n");

1096

return false;

1097

}

1098

if (gimple_call_fndecl (stmt) != NULL_TREE)

1099

{

1100

if (dump_file)

1101

fprintf (dump_file, "Call is already direct\n");

1102

return false;

1103

}

1016

1104

1017

gcov_type total = 0;

1105

gcov_type total = 0;

1018

icall_target_map::const_iterator max_iter = map.end ();

1106

icall_target_map::const_iterator max_iter = map.end ();

@@ -1026,38 +1114,47 @@ afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,

1026

}

1114

}

1027

struct cgraph_node *direct_call = cgraph_node::get_for_asmname (

1115

struct cgraph_node *direct_call = cgraph_node::get_for_asmname (

1028

get_identifier (afdo_string_table->get_name (max_iter->first)));

1116

get_identifier (afdo_string_table->get_name (max_iter->first)));

1029

if (direct_call == NULL || !direct_call->profile_id)

1117

if (direct_call == NULL)

1030

return false;

1118

{

1119

if (dump_file)

1120

fprintf (dump_file, "Failed to find cgraph node for %s\n",

1121

afdo_string_table->get_name (max_iter->first));

1122

return false;

1123

}

1031

1124

1032

callee = gimple_call_fn (stmt);

1125

callee = gimple_call_fn (stmt);

1033

1126

1034

histogram_value hist = gimple_alloc_histogram_value (

1035

cfun, HIST_TYPE_INDIR_CALL, stmt, callee);

1036

hist->n_counters = 4;

1037

hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);

1038

gimple_add_histogram_value (cfun, stmt, hist);

1039

1040

/* Total counter */

1041

hist->hvalue.counters[0] = total;

1042

/* Number of value/counter pairs */

1043

hist->hvalue.counters[1] = 1;

1044

/* Value */

1045

hist->hvalue.counters[2] = direct_call->profile_id;

1046

/* Counter */

1047

hist->hvalue.counters[3] = max_iter->second;

1048

1049

if (!transform)

1127

if (!transform)

1050

return false;

1128

{

1051

1129

if (!direct_call->profile_id)

1052

cgraph_node* current_function_node = cgraph_node::get (current_function_decl);

1130

{

1053

1131

if (dump_file)

1054

/* If the direct call is a recursive call, don't promote it since

1132

fprintf (dump_file, "No profile id\n");

1055

we are not set up to inline recursive calls at this stage. */

1133

return false;

1056

if (direct_call == current_function_node)

1134

}

1057

return false;

1135

histogram_value hist = gimple_alloc_histogram_value (

1058

1136

cfun, HIST_TYPE_INDIR_CALL, stmt, callee);

1059

struct cgraph_edge *indirect_edge

1137

hist->n_counters = 4;

1060

= current_function_node->get_edge (stmt);

1138

hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);

1139

gimple_add_histogram_value (cfun, stmt, hist);

1140

1141

/* Total counter */

1142

hist->hvalue.counters[0] = total;

1143

/* Number of value/counter pairs */

1144

hist->hvalue.counters[1] = 1;

1145

/* Value */

1146

hist->hvalue.counters[2] = direct_call->profile_id;

1147

/* Counter */

1148

hist->hvalue.counters[3] = max_iter->second;

1149

1150

if (!direct_call->profile_id)

1151

{

1152

if (dump_file)

1153

fprintf (dump_file, "Histogram attached\n");

1154

return false;

1155

}

1156

return false;

1157

}

1061

1158

1062

if (dump_file)

1159

if (dump_file)

1063

{

1160

{

@@ -1067,16 +1164,10 @@ afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,

1067

print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);

1164

print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);

1068

}

1165

}

1069

1166

1070

if (direct_call == NULL)

1167

if (!direct_call->definition)

1071

{

1072

if (dump_file)

1073

fprintf (dump_file, " not transforming\n");

1074

return false;

1075

}

1076

if (DECL_STRUCT_FUNCTION (direct_call->decl) == NULL)

1077

{

1168

{

1078

if (dump_file)

1169

if (dump_file)

1079

fprintf (dump_file, " no declaration\n");

1170

fprintf (dump_file, " no definition available\n");

1080

return false;

1171

return false;

1081

}

1172

}

1082

1173

@@ -1087,13 +1178,9 @@ afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,

1087

fprintf (dump_file, "\n");

1178

fprintf (dump_file, "\n");

1088

}

1179

}

1089

1180

1090

struct cgraph_edge *new_edge

1181

indirect_edge->make_speculative

1091

= indirect_edge->make_speculative

1092

(direct_call,

1182

(direct_call,

1093

gimple_bb (stmt)->count.apply_scale (99, 100));

1183

gimple_bb (stmt)->count.apply_scale (99, 100));

1094

cgraph_edge::redirect_call_stmt_to_callee (new_edge);

1095

gimple_remove_histogram_value (cfun, stmt, hist);

1096

inline_call (new_edge, true, NULL, NULL, false);

1097

return true;

1184

return true;

1098

}

1185

}

1099

1186

@@ -1101,10 +1188,10 @@ afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map,

1101

histograms and adds them to list VALUES. */

1188

histograms and adds them to list VALUES. */

1102

1189

1103

static bool

1190

static bool

1104

afdo_vpt (gimple_stmt_iterator *gsi, const icall_target_map &map,

1191

afdo_vpt (gcall *gs, const icall_target_map &map,

1105

bool transform)

1192

bool transform, cgraph_edge *indirect_edge)

1106

{

1193

{

1107

return afdo_indirect_call (gsi, map, transform);

1194

return afdo_indirect_call (gs, map, transform, indirect_edge);

1108

}

1195

}

1109

1196

1110

typedef std::set<basic_block> bb_set;

1197

typedef std::set<basic_block> bb_set;

@@ -1150,7 +1237,7 @@ update_count_by_afdo_count (profile_count *count, gcov_type c)

1150

Return TRUE if BB is annotated. */

1237

Return TRUE if BB is annotated. */

1151

1238

1152

static bool

1239

static bool

1153

afdo_set_bb_count (basic_block bb, const stmt_set &promoted)

1240

afdo_set_bb_count (basic_block bb)

1154

{

1241

{

1155

gimple_stmt_iterator gsi;

1242

gimple_stmt_iterator gsi;

1156

gcov_type max_count = 0;

1243

gcov_type max_count = 0;

@@ -1163,22 +1250,26 @@ afdo_set_bb_count (basic_block bb, const stmt_set &promoted)

1163

count_info info;

1250

count_info info;

1164

gimple *stmt = gsi_stmt (gsi);

1251

gimple *stmt = gsi_stmt (gsi);

1165

if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))

1252

if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))

1166

continue;

1253

continue;

1167

if (afdo_source_profile->get_count_info (stmt, &info))

1254

if (afdo_source_profile->get_count_info (stmt, &info))

1168

{

1255

{

1169

if (info.count > max_count)

1256

if (info.count > max_count)

1170

max_count = info.count;

1257

max_count = info.count;

1171

if (dump_file && info.count)

1258

if (dump_file && info.count)

1172

{

1259

{

1173

fprintf (dump_file, " count %" PRIu64 " in stmt: ",

1260

fprintf (dump_file, " count %" PRIu64 " in stmt: ",

1174

(int64_t)info.count);

1261

(int64_t)info.count);

1175

print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);

1262

print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);

1176

}

1263

}

1177

has_annotated = true;

1264

has_annotated = true;

1178

if (info.targets.size () > 0

1265

gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));

1179

&& promoted.find (stmt) == promoted.end ())

1266

/* TODO; if inlined early and indirect call was not optimized out,

1180

afdo_vpt (&gsi, info.targets, false);

1267

we will end up speculating again. Early inliner should remove

1181

}

1268

all targets for edges it speculated into safely. */

1269

if (call

1270

&& info.targets.size () > 0)

1271

afdo_vpt (call, info.targets, false, NULL);

1272

}

1182

}

1273

}

1183

1274

1184

if (!has_annotated)

1275

if (!has_annotated)

@@ -1899,76 +1990,10 @@ afdo_calculate_branch_prob (bb_set *annotated_bb)

1899

}

1990

}

1900

}

1991

}

1901

1992

1902

/* Perform value profile transformation using AutoFDO profile. Add the

1993

/* Annotate auto profile to the control flow graph. */

1903

promoted stmts to PROMOTED_STMTS. Return TRUE if there is any

1904

indirect call promoted. */

1905

1906

static bool

1907

afdo_vpt_for_early_inline (stmt_set *promoted_stmts)

1908

{

1909

basic_block bb;

1910

if (afdo_source_profile->get_function_instance_by_decl (

1911

current_function_decl) == NULL)

1912

return false;

1913

1914

compute_fn_summary (cgraph_node::get (current_function_decl), true);

1915

1916

bool has_vpt = false;

1917

FOR_EACH_BB_FN (bb, cfun)

1918

{

1919

if (!has_indirect_call (bb))

1920

continue;

1921

gimple_stmt_iterator gsi;

1922

1923

gcov_type bb_count = 0;

1924

for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))

1925

{

1926

count_info info;

1927

gimple *stmt = gsi_stmt (gsi);

1928

if (afdo_source_profile->get_count_info (stmt, &info))

1929

bb_count = MAX (bb_count, info.count);

1930

}

1931

1932

for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))

1933

{

1934

gcall *stmt = dyn_cast <gcall *> (gsi_stmt (gsi));

1935

/* IC_promotion and early_inline_2 is done in multiple iterations.

1936

No need to promoted the stmt if its in promoted_stmts (means

1937

it is already been promoted in the previous iterations). */

1938

if ((!stmt) || gimple_call_fn (stmt) == NULL

1939

|| TREE_CODE (gimple_call_fn (stmt)) == FUNCTION_DECL

1940

|| promoted_stmts->find (stmt) != promoted_stmts->end ())

1941

continue;

1942

1943

count_info info;

1944

afdo_source_profile->get_count_info (stmt, &info);

1945

info.count = bb_count;

1946

if (afdo_source_profile->update_inlined_ind_target (stmt, &info))

1947

{

1948

/* Promote the indirect call and update the promoted_stmts. */

1949

promoted_stmts->insert (stmt);

1950

if (afdo_vpt (&gsi, info.targets, true))

1951

has_vpt = true;

1952

}

1953

}

1954

}

1955

1956

if (has_vpt)

1957

{

1958

unsigned todo = optimize_inline_calls (current_function_decl);

1959

if (todo & TODO_update_ssa_any)

1960

update_ssa (TODO_update_ssa);

1961

return true;

1962

}

1963

1964

return false;

1965

}

1966

1967

/* Annotate auto profile to the control flow graph. Do not annotate value

1968

profile for stmts in PROMOTED_STMTS. */

1969

1994

1970

static void

1995

static void

1971

afdo_annotate_cfg (const stmt_set &promoted_stmts)

1996

afdo_annotate_cfg (void)

1972

{

1997

{

1973

basic_block bb;

1998

basic_block bb;

1974

bb_set annotated_bb;

1999

bb_set annotated_bb;

@@ -1997,7 +2022,7 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts)

1997

bool profile_found = head_count > 0;

2022

bool profile_found = head_count > 0;

1998

FOR_EACH_BB_FN (bb, cfun)

2023

FOR_EACH_BB_FN (bb, cfun)

1999

{

2024

{

2000

if (afdo_set_bb_count (bb, promoted_stmts))

2025

if (afdo_set_bb_count (bb))

2001

{

2026

{

2002

if (bb->count.quality () == AFDO)

2027

if (bb->count.quality () == AFDO)

2003

{

2028

{

@@ -2125,45 +2150,8 @@ auto_profile (void)

2125

2150

2126

push_cfun (DECL_STRUCT_FUNCTION (node->decl));

2151

push_cfun (DECL_STRUCT_FUNCTION (node->decl));

2127

2152

2128

/* First do indirect call promotion and early inline to make the

2153

unsigned int todo = early_inline ();

2129

IR match the profiled binary before actual annotation.

2154

autofdo::afdo_annotate_cfg ();

2130

2131

This is needed because an indirect call might have been promoted

2132

and inlined in the profiled binary. If we do not promote and

2133

inline these indirect calls before annotation, the profile for

2134

these promoted functions will be lost.

2135

2136

e.g. foo() --indirect_call--> bar()

2137

In profiled binary, the callsite is promoted and inlined, making

2138

the profile look like:

2139

2140

foo: {

2141

loc_foo_1: count_1

2142

bar@loc_foo_2: {

2143

loc_bar_1: count_2

2144

loc_bar_2: count_3

2145

}

2146

}

2147

2148

Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.

2149

If we perform annotation on it, the profile inside bar@loc_foo2

2150

will be wasted.

2151

2152

To avoid this, we promote loc_foo_2 and inline the promoted bar

2153

function before annotation, so the profile inside bar@loc_foo2

2154

will be useful. */

2155

autofdo::stmt_set promoted_stmts;

2156

unsigned int todo = 0;

2157

for (int i = 0; i < 10; i++)

2158

{

2159

if (!flag_value_profile_transformations

2160

|| !autofdo::afdo_vpt_for_early_inline (&promoted_stmts))

2161

break;

2162

todo |= early_inline ();

2163

}

2164

2165

todo |= early_inline ();

2166

autofdo::afdo_annotate_cfg (promoted_stmts);

2167

compute_function_frequency ();

2155

compute_function_frequency ();

2168

2156

2169

/* Local pure-const may imply need to fixup the cfg. */

2157

/* Local pure-const may imply need to fixup the cfg. */

@@ -2225,6 +2213,14 @@ afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge)

2225

temporarily set it to afdo_profile_info to calculate hotness. */

2213

temporarily set it to afdo_profile_info to calculate hotness. */

2226

profile_info = autofdo::afdo_profile_info;

2214

profile_info = autofdo::afdo_profile_info;

2227

is_hot = maybe_hot_count_p (NULL, pcount);

2215

is_hot = maybe_hot_count_p (NULL, pcount);

2216

if (dump_file)

2217

{

2218

fprintf (dump_file, "Call %s -> %s has %s afdo profile count ",

2219

edge->caller->dump_name (), edge->callee->dump_name (),

2220

is_hot ? "hot" : "cold");

2221

pcount.dump (dump_file);

2222

fprintf (dump_file, "\n");

2223

}

2228

profile_info = saved_profile_info;

2224

profile_info = saved_profile_info;

2229

return is_hot;

2225

return is_hot;

2230

}

2226

}

@@ -2232,6 +2228,78 @@ afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge)

2232

return false;

2228

return false;

2233

}

2229

}

2234

2230

2231

/* Do indirect call promotion during early inlining to make the

2232

IR match the profiled binary before actual annotation.

2233

2234

This is needed because an indirect call might have been promoted

2235

and inlined in the profiled binary. If we do not promote and

2236

inline these indirect calls before annotation, the profile for

2237

these promoted functions will be lost.

2238

2239

e.g. foo() --indirect_call--> bar()

2240

In profiled binary, the callsite is promoted and inlined, making

2241

the profile look like:

2242

2243

foo: {

2244

loc_foo_1: count_1

2245

bar@loc_foo_2: {

2246

loc_bar_1: count_2

2247

loc_bar_2: count_3

2248

}

2249

}

2250

2251

Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.

2252

If we perform annotation on it, the profile inside bar@loc_foo2

2253

will be wasted.

2254

2255

To avoid this, we promote loc_foo_2 and inline the promoted bar

2256

function before annotation, so the profile inside bar@loc_foo2

2257

will be useful. */

2258

2259

bool

2260

afdo_vpt_for_early_inline (cgraph_node *node)

2261

{

2262

if (!node->indirect_calls)

2263

return false;

2264

bool changed = false;

2265

cgraph_node *outer = node->inlined_to ? node->inlined_to : node;

2266

if (autofdo::afdo_source_profile->get_function_instance_by_decl

2267

(outer->decl) == NULL)

2268

return false;

2269

for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)

2270

{

2271

gcov_type bb_count = 0;

2272

autofdo::count_info info;

2273

basic_block bb = gimple_bb (e->call_stmt);

2274

2275

/* TODO: This is quadratic; cache the value. */

2276

for (gimple_stmt_iterator gsi = gsi_start_bb (bb);

2277

!gsi_end_p (gsi); gsi_next (&gsi))

2278

{

2279

autofdo::count_info info;

2280

gimple *stmt = gsi_stmt (gsi);

2281

if (autofdo::afdo_source_profile->get_count_info (stmt, &info, node))

2282

bb_count = MAX (bb_count, info.count);

2283

}

2284

autofdo::afdo_source_profile->get_count_info (e->call_stmt, &info, node);

2285

info.count = bb_count;

2286

if (!autofdo::afdo_source_profile->update_inlined_ind_target

2287

(e->call_stmt, &info, node))

2288

continue;

2289

changed |= autofdo::afdo_vpt (e->call_stmt, info.targets, true, e);

2290

}

2291

return changed;

2292

}

2293

2294

/* If speculation used during early inline, remove the target

2295

so we do not speculate the indirect edge again during afdo pass. */

2296

2297

void

2298

remove_afdo_speculative_target (cgraph_edge *e)

2299

{

2300

autofdo::afdo_source_profile->remove_icall_target (e);

2301

}

2302

2235

namespace

2303

namespace

2236

{

2304

{

2237

2305

@@ -28,4 +28,11 @@ extern void end_auto_profile (void);

28

/* Returns TRUE if EDGE is hot enough to be inlined early. */

28

/* Returns TRUE if EDGE is hot enough to be inlined early. */

29

extern bool afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *);

29

extern bool afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *);

30

30

31

/* Try to turn indirect calls into speculative calls for early inlining. */

32

extern bool afdo_vpt_for_early_inline (cgraph_node *node);

33

34

/* If speculation was early inlined, remove it from profile data so we

35

do not repeat it later. */

36

extern void remove_afdo_speculative_target (cgraph_edge *);

37

31

#endif /* AUTO_PROFILE_H */

38

#endif /* AUTO_PROFILE_H */

@@ -3113,29 +3113,39 @@ early_inline_small_functions (struct cgraph_node *node)

3113

and inlining seems useful. That is there are enough samples in the callee

3113

and inlining seems useful. That is there are enough samples in the callee

3114

function.

3114

function.

3115

3115

3116

Unlike early inlining, we inline recursively.

3116

Unlike early inlining, we inline recursively. Profile data is also used

3117

TODO: We should also integrate VPT. */

3117

to produce speculative calls which we then inline. In the case some

3118

speculatin was introduced, set SPECULATIVE_CALLS. */

3118

3119

3119

static bool

3120

static bool

3120

inline_functions_by_afdo (struct cgraph_node *node)

3121

inline_functions_by_afdo (struct cgraph_node *node, bool *speculative_calls)

3121

{

3122

{

3122

if (!flag_auto_profile)

3123

if (!flag_auto_profile)

3123

return false;

3124

return false;

3124

struct cgraph_edge *e;

3125

struct cgraph_edge *e;

3125

bool inlined = false;

3126

bool inlined = false;

3126

3127

3127

for (e = node->callees; e; e = e->next_callee)

3128

*speculative_calls |= afdo_vpt_for_early_inline (node);

3129

3130

cgraph_edge *next;

3131

for (e = node->callees; e; e = next)

3128

{

3132

{

3129

struct cgraph_node *callee = e->callee->ultimate_alias_target ();

3133

next = e->next_callee;

3130

3134

3131

if (!e->inline_failed)

3135

if (!e->inline_failed)

3132

{

3136

{

3133

inlined |= inline_functions_by_afdo (e->callee);

3137

inlined |= inline_functions_by_afdo (e->callee, speculative_calls);

3134

continue;

3138

continue;

3135

}

3139

}

3136

if (!afdo_callsite_hot_enough_for_early_inline (e))

3140

if (!afdo_callsite_hot_enough_for_early_inline (e))

3137

continue;

3141

{

3142

/* If we do not want to inline, remove the speculation. */

3143

if (e->speculative)

3144

cgraph_edge::resolve_speculation (e);

3145

continue;

3146

}

3138

3147

3148

struct cgraph_node *callee = e->callee->ultimate_alias_target ();

3139

if (callee->definition

3149

if (callee->definition

3140

&& !ipa_fn_summaries->get (callee))

3150

&& !ipa_fn_summaries->get (callee))

3141

compute_fn_summary (callee, true);

3151

compute_fn_summary (callee, true);

@@ -3147,6 +3157,9 @@ inline_functions_by_afdo (struct cgraph_node *node)

3147

"Not inlining %C -> %C using auto-profile, %s.",

3157

"Not inlining %C -> %C using auto-profile, %s.",

3148

e->caller, e->callee,

3158

e->caller, e->callee,

3149

cgraph_inline_failed_string (e->inline_failed));

3159

cgraph_inline_failed_string (e->inline_failed));

3160

/* If we do not want to inline, remove the speculation. */

3161

if (e->speculative)

3162

cgraph_edge::resolve_speculation (e);

3150

continue;

3163

continue;

3151

}

3164

}

3152

/* We can handle recursive inlining by first producing

3165

/* We can handle recursive inlining by first producing

@@ -3158,6 +3171,9 @@ inline_functions_by_afdo (struct cgraph_node *node)

3158

"Not inlining %C recursively"

3171

"Not inlining %C recursively"

3159

" using auto-profile.\n",

3172

" using auto-profile.\n",

3160

e->callee);

3173

e->callee);

3174

/* If we do not want to inline, remove the speculation. */

3175

if (e->speculative)

3176

cgraph_edge::resolve_speculation (e);

3161

continue;

3177

continue;

3162

}

3178

}

3163

3179

@@ -3173,8 +3189,10 @@ inline_functions_by_afdo (struct cgraph_node *node)

3173

"Inlining using auto-profile %C into %C.\n",

3189

"Inlining using auto-profile %C into %C.\n",

3174

callee, e->caller);

3190

callee, e->caller);

3175

}

3191

}

3192

if (e->speculative)

3193

remove_afdo_speculative_target (e);

3176

inline_call (e, true, NULL, NULL, false);

3194

inline_call (e, true, NULL, NULL, false);

3177

inlined |= inline_functions_by_afdo (e->callee);

3195

inlined |= inline_functions_by_afdo (e->callee, speculative_calls);

3178

inlined = true;

3196

inlined = true;

3179

}

3197

}

3180

3198

@@ -3262,10 +3280,20 @@ early_inliner (function *fun)

3262

param_early_inliner_max_iterations))

3280

param_early_inliner_max_iterations))

3263

{

3281

{

3264

bool inlined = early_inline_small_functions (node);

3282

bool inlined = early_inline_small_functions (node);

3265

inlined |= inline_functions_by_afdo (node);

3283

bool speculative_calls = false;

3284

inlined |= inline_functions_by_afdo (node, &speculative_calls);

3266

if (!inlined)

3285

if (!inlined)

3267

break;

3286

break;

3268

timevar_push (TV_INTEGRATION);

3287

timevar_push (TV_INTEGRATION);

3288

if (speculative_calls)

3289

{

3290

cgraph_edge *next;

3291

for (cgraph_edge *e = node->callees; e; e = next)

3292

{

3293

next = e->next_callee;

3294

cgraph_edge::redirect_call_stmt_to_callee (e);

3295

}

3296

}

3269

todo |= optimize_inline_calls (current_function_decl);

3297

todo |= optimize_inline_calls (current_function_decl);

3270

3298

3271

/* Technically we ought to recompute inline parameters so the new

3299

/* Technically we ought to recompute inline parameters so the new

@@ -13,20 +13,26 @@ struct wrapptr

13

int test (struct wrapptr *p)

13

int test (struct wrapptr *p)

14

{

14

{

15

int s = 0;

15

int s = 0;

16

int (*ret)(int) = p->ret;

16

for (int pos = 0; pos < 1000; pos++)

17

for (int pos = 0; pos < 1000; pos++)

17

s+p->ret(pos);

18

ret(pos);

18

if (s)

19

if (s)

19

__builtin_printf ("sum error\n");

20

__builtin_printf ("sum error\n");

20

}

21

}

21

int main()

22

int main()

22

{

23

{

23

struct wrapptr p={reta};

24

for (int i = 0; i < 10000; i++)

24

for (int i = 0; i < 10000; i++)

25

{

26

struct wrapptr p={reta};

27

28

29

25

test(&p);

30

test(&p);

31

}

26

return 0;

32

return 0;

27

}

33

}

28

/* { dg-final-use-autofdo { scan-tree-dump "Inlining using auto-profile test" "einline"} } */

34

/* { dg-final-use-autofdo { scan-tree-dump "Inlining using auto-profile test" "einline"} } */

29

/* { dg-final-use-autofdo { scan-ipa-dump "Checking indirect call -> direct call reta" "afdo"} } */

35

/* { dg-final-use-autofdo { scan-tree-dump "Checking indirect call -> direct call ret_" "einline"} } */

30

/* { dg-final-use-autofdo { scan-ipa-dump-times "looks good" 0 "afdo"} } */

36

/* { dg-final-use-autofdo { scan-tree-dump "looks good" "einline"} } */

31

/* If we inlined reta->test->main, it will contian array[pos]. */

37

/* If we inlined reta->test->main, it will contian array[pos]. */

32

/* { dg-final-use-autofdo { scan-ipa-dump "array.pos_" "afdo"} } */

38

/* { dg-final-use-autofdo { scan-tree-dump "array.pos_" "einline"} } */

@@ -31,5 +31,5 @@ main (void)

31

}

31

}

32

/* { dg-final-use-not-autofdo { scan-ipa-dump "Indirect call -> direct call.* add1 .will resolve by ipa-profile" "profile"} } */

32

/* { dg-final-use-not-autofdo { scan-ipa-dump "Indirect call -> direct call.* add1 .will resolve by ipa-profile" "profile"} } */

33

/* { dg-final-use-not-autofdo { scan-ipa-dump "Indirect call -> direct call.* sub1 .will resolve by ipa-profile" "profile"} } */

33

/* { dg-final-use-not-autofdo { scan-ipa-dump "Indirect call -> direct call.* sub1 .will resolve by ipa-profile" "profile"} } */

34

/* { dg-final-use-autofdo { scan-ipa-dump "Inlining add1/. into main/" "afdo"} } */

34

/* { dg-final-use-autofdo { scan-tree-dump "Inlining add1/. into main/" "einline"} } */

35

/* { dg-final-use-autofdo { scan-ipa-dump "Inlining sub1/. into main/" "afdo"} } */

35

/* { dg-final-use-autofdo { scan-tree-dump "Inlining sub1/. into main/" "einline"} } */

@@ -4846,7 +4846,9 @@ expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id,

4846

goto egress;

4846

goto egress;

4847

4847

4848

cg_edge = id->dst_node->get_edge (stmt);

4848

cg_edge = id->dst_node->get_edge (stmt);

4849

gcc_checking_assert (cg_edge);

4849

/* Edge should exist and speculations should be resolved at this

4850

stage. */

4851

gcc_checking_assert (cg_edge && !cg_edge->speculative);

4850

/* First, see if we can figure out what function is being called.

4852

/* First, see if we can figure out what function is being called.

4851

If we cannot, then there is no hope of inlining the function. */

4853

If we cannot, then there is no hope of inlining the function. */

4852

if (cg_edge->indirect_unknown_callee)

4854

if (cg_edge->indirect_unknown_callee)