673 int dd_main_exit = dom_depth(main_exit);
674
675 // Step A1: Clone the loop body. The clone becomes the post-loop. The main
676 // loop pre-header illegally has 2 control users (old & new loops).
677 clone_loop( loop, old_new, dd_main_exit );
678 assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
679 CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
680 post_head->set_post_loop(main_head);
681
682 // Build the main-loop normal exit.
683 IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
684 _igvn.register_new_node_with_optimizer( new_main_exit );
685 set_idom(new_main_exit, main_end, dd_main_exit );
686 set_loop(new_main_exit, loop->_parent);
687
688 // Step A2: Build a zero-trip guard for the post-loop. After leaving the
689 // main-loop, the post-loop may not execute at all. We 'opaque' the incr
690 // (the main-loop trip-counter exit value) because we will be changing
691 // the exit value (via unrolling) so we cannot constant-fold away the zero
692 // trip guard until all unrolling is done.
693 Node *zer_opaq = new (C, 2) Opaque1Node(incr);
694 Node *zer_cmp = new (C, 3) CmpINode( zer_opaq, limit );
695 Node *zer_bol = new (C, 2) BoolNode( zer_cmp, b_test );
696 register_new_node( zer_opaq, new_main_exit );
697 register_new_node( zer_cmp , new_main_exit );
698 register_new_node( zer_bol , new_main_exit );
699
700 // Build the IfNode
701 IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
702 _igvn.register_new_node_with_optimizer( zer_iff );
703 set_idom(zer_iff, new_main_exit, dd_main_exit);
704 set_loop(zer_iff, loop->_parent);
705
706 // Plug in the false-path, taken if we need to skip post-loop
707 _igvn.hash_delete( main_exit );
708 main_exit->set_req(0, zer_iff);
709 _igvn._worklist.push(main_exit);
710 set_idom(main_exit, zer_iff, dd_main_exit);
711 set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
712 // Make the true-path, must enter the post loop
713 Node *zer_taken = new (C, 1) IfTrueNode( zer_iff );
743 // Step B1: Clone the loop body. The clone becomes the pre-loop. The main
744 // loop pre-header illegally has 2 control users (old & new loops).
745 clone_loop( loop, old_new, dd_main_head );
746 CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop();
747 CountedLoopEndNode* pre_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
748 pre_head->set_pre_loop(main_head);
749 Node *pre_incr = old_new[incr->_idx];
750
751 // Find the pre-loop normal exit.
752 Node* pre_exit = pre_end->proj_out(false);
753 assert( pre_exit->Opcode() == Op_IfFalse, "" );
754 IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
755 _igvn.register_new_node_with_optimizer( new_pre_exit );
756 set_idom(new_pre_exit, pre_end, dd_main_head);
757 set_loop(new_pre_exit, loop->_parent);
758
759 // Step B2: Build a zero-trip guard for the main-loop. After leaving the
760 // pre-loop, the main-loop may not execute at all. Later in life this
761 // zero-trip guard will become the minimum-trip guard when we unroll
762 // the main-loop.
763 Node *min_opaq = new (C, 2) Opaque1Node(limit);
764 Node *min_cmp = new (C, 3) CmpINode( pre_incr, min_opaq );
765 Node *min_bol = new (C, 2) BoolNode( min_cmp, b_test );
766 register_new_node( min_opaq, new_pre_exit );
767 register_new_node( min_cmp , new_pre_exit );
768 register_new_node( min_bol , new_pre_exit );
769
770 // Build the IfNode
771 IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_FAIR, COUNT_UNKNOWN );
772 _igvn.register_new_node_with_optimizer( min_iff );
773 set_idom(min_iff, new_pre_exit, dd_main_head);
774 set_loop(min_iff, loop->_parent);
775
776 // Plug in the false-path, taken if we need to skip main-loop
777 _igvn.hash_delete( pre_exit );
778 pre_exit->set_req(0, min_iff);
779 set_idom(pre_exit, min_iff, dd_main_head);
780 set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
781 // Make the true-path, must enter the main loop
782 Node *min_taken = new (C, 1) IfTrueNode( min_iff );
783 _igvn.register_new_node_with_optimizer( min_taken );
793 for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
794 Node* main_phi = main_head->fast_out(i2);
795 if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
796 Node *pre_phi = old_new[main_phi->_idx];
797 Node *fallpre = clone_up_backedge_goo(pre_head->back_control(),
798 main_head->init_control(),
799 pre_phi->in(LoopNode::LoopBackControl));
800 _igvn.hash_delete(main_phi);
801 main_phi->set_req( LoopNode::EntryControl, fallpre );
802 }
803 }
804
805 // Step B4: Shorten the pre-loop to run only 1 iteration (for now).
806 // RCE and alignment may change this later.
807 Node *cmp_end = pre_end->cmp_node();
808 assert( cmp_end->in(2) == limit, "" );
809 Node *pre_limit = new (C, 3) AddINode( init, stride );
810
811 // Save the original loop limit in this Opaque1 node for
812 // use by range check elimination.
813 Node *pre_opaq = new (C, 3) Opaque1Node(pre_limit, limit);
814
815 register_new_node( pre_limit, pre_head->in(0) );
816 register_new_node( pre_opaq , pre_head->in(0) );
817
818 // Since no other users of pre-loop compare, I can hack limit directly
819 assert( cmp_end->outcnt() == 1, "no other users" );
820 _igvn.hash_delete(cmp_end);
821 cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
822
823 // Special case for not-equal loop bounds:
824 // Change pre loop test, main loop test, and the
825 // main loop guard test to use lt or gt depending on stride
826 // direction:
827 // positive stride use <
828 // negative stride use >
829
830 if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
831
832 BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
833 // Modify pre loop end condition
|
673 int dd_main_exit = dom_depth(main_exit);
674
675 // Step A1: Clone the loop body. The clone becomes the post-loop. The main
676 // loop pre-header illegally has 2 control users (old & new loops).
677 clone_loop( loop, old_new, dd_main_exit );
678 assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
679 CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
680 post_head->set_post_loop(main_head);
681
682 // Build the main-loop normal exit.
683 IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
684 _igvn.register_new_node_with_optimizer( new_main_exit );
685 set_idom(new_main_exit, main_end, dd_main_exit );
686 set_loop(new_main_exit, loop->_parent);
687
688 // Step A2: Build a zero-trip guard for the post-loop. After leaving the
689 // main-loop, the post-loop may not execute at all. We 'opaque' the incr
690 // (the main-loop trip-counter exit value) because we will be changing
691 // the exit value (via unrolling) so we cannot constant-fold away the zero
692 // trip guard until all unrolling is done.
693 Node *zer_opaq = new (C, 2) Opaque1Node(C, incr);
694 Node *zer_cmp = new (C, 3) CmpINode( zer_opaq, limit );
695 Node *zer_bol = new (C, 2) BoolNode( zer_cmp, b_test );
696 register_new_node( zer_opaq, new_main_exit );
697 register_new_node( zer_cmp , new_main_exit );
698 register_new_node( zer_bol , new_main_exit );
699
700 // Build the IfNode
701 IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
702 _igvn.register_new_node_with_optimizer( zer_iff );
703 set_idom(zer_iff, new_main_exit, dd_main_exit);
704 set_loop(zer_iff, loop->_parent);
705
706 // Plug in the false-path, taken if we need to skip post-loop
707 _igvn.hash_delete( main_exit );
708 main_exit->set_req(0, zer_iff);
709 _igvn._worklist.push(main_exit);
710 set_idom(main_exit, zer_iff, dd_main_exit);
711 set_idom(main_exit->unique_out(), zer_iff, dd_main_exit);
712 // Make the true-path, must enter the post loop
713 Node *zer_taken = new (C, 1) IfTrueNode( zer_iff );
743 // Step B1: Clone the loop body. The clone becomes the pre-loop. The main
744 // loop pre-header illegally has 2 control users (old & new loops).
745 clone_loop( loop, old_new, dd_main_head );
746 CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop();
747 CountedLoopEndNode* pre_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
748 pre_head->set_pre_loop(main_head);
749 Node *pre_incr = old_new[incr->_idx];
750
751 // Find the pre-loop normal exit.
752 Node* pre_exit = pre_end->proj_out(false);
753 assert( pre_exit->Opcode() == Op_IfFalse, "" );
754 IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
755 _igvn.register_new_node_with_optimizer( new_pre_exit );
756 set_idom(new_pre_exit, pre_end, dd_main_head);
757 set_loop(new_pre_exit, loop->_parent);
758
759 // Step B2: Build a zero-trip guard for the main-loop. After leaving the
760 // pre-loop, the main-loop may not execute at all. Later in life this
761 // zero-trip guard will become the minimum-trip guard when we unroll
762 // the main-loop.
763 Node *min_opaq = new (C, 2) Opaque1Node(C, limit);
764 Node *min_cmp = new (C, 3) CmpINode( pre_incr, min_opaq );
765 Node *min_bol = new (C, 2) BoolNode( min_cmp, b_test );
766 register_new_node( min_opaq, new_pre_exit );
767 register_new_node( min_cmp , new_pre_exit );
768 register_new_node( min_bol , new_pre_exit );
769
770 // Build the IfNode
771 IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_FAIR, COUNT_UNKNOWN );
772 _igvn.register_new_node_with_optimizer( min_iff );
773 set_idom(min_iff, new_pre_exit, dd_main_head);
774 set_loop(min_iff, loop->_parent);
775
776 // Plug in the false-path, taken if we need to skip main-loop
777 _igvn.hash_delete( pre_exit );
778 pre_exit->set_req(0, min_iff);
779 set_idom(pre_exit, min_iff, dd_main_head);
780 set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
781 // Make the true-path, must enter the main loop
782 Node *min_taken = new (C, 1) IfTrueNode( min_iff );
783 _igvn.register_new_node_with_optimizer( min_taken );
793 for (DUIterator_Fast i2max, i2 = main_head->fast_outs(i2max); i2 < i2max; i2++) {
794 Node* main_phi = main_head->fast_out(i2);
795 if( main_phi->is_Phi() && main_phi->in(0) == main_head && main_phi->outcnt() > 0 ) {
796 Node *pre_phi = old_new[main_phi->_idx];
797 Node *fallpre = clone_up_backedge_goo(pre_head->back_control(),
798 main_head->init_control(),
799 pre_phi->in(LoopNode::LoopBackControl));
800 _igvn.hash_delete(main_phi);
801 main_phi->set_req( LoopNode::EntryControl, fallpre );
802 }
803 }
804
805 // Step B4: Shorten the pre-loop to run only 1 iteration (for now).
806 // RCE and alignment may change this later.
807 Node *cmp_end = pre_end->cmp_node();
808 assert( cmp_end->in(2) == limit, "" );
809 Node *pre_limit = new (C, 3) AddINode( init, stride );
810
811 // Save the original loop limit in this Opaque1 node for
812 // use by range check elimination.
813 Node *pre_opaq = new (C, 3) Opaque1Node(C, pre_limit, limit);
814
815 register_new_node( pre_limit, pre_head->in(0) );
816 register_new_node( pre_opaq , pre_head->in(0) );
817
818 // Since no other users of pre-loop compare, I can hack limit directly
819 assert( cmp_end->outcnt() == 1, "no other users" );
820 _igvn.hash_delete(cmp_end);
821 cmp_end->set_req(2, peel_only ? pre_limit : pre_opaq);
822
823 // Special case for not-equal loop bounds:
824 // Change pre loop test, main loop test, and the
825 // main loop guard test to use lt or gt depending on stride
826 // direction:
827 // positive stride use <
828 // negative stride use >
829
830 if (pre_end->in(CountedLoopEndNode::TestValue)->as_Bool()->_test._test == BoolTest::ne) {
831
832 BoolTest::mask new_test = (main_end->stride_con() > 0) ? BoolTest::lt : BoolTest::gt;
833 // Modify pre loop end condition
|