662 if( cmp->outcnt() != 1 ) {
663 cmp = cmp->clone();
664 register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
665 _igvn.hash_delete(bol);
666 bol->set_req(1, cmp);
667 }
668
669 //------------------------------
670 // Step A: Create Post-Loop.
671 Node* main_exit = main_end->proj_out(false);
672 assert( main_exit->Opcode() == Op_IfFalse, "" );
673 int dd_main_exit = dom_depth(main_exit);
674
675 // Step A1: Clone the loop body. The clone becomes the post-loop. The main
676 // loop pre-header illegally has 2 control users (old & new loops).
677 clone_loop( loop, old_new, dd_main_exit );
678 assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
679 CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
680 post_head->set_post_loop(main_head);
681
682 // Build the main-loop normal exit.
683 IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
684 _igvn.register_new_node_with_optimizer( new_main_exit );
685 set_idom(new_main_exit, main_end, dd_main_exit );
686 set_loop(new_main_exit, loop->_parent);
687
688 // Step A2: Build a zero-trip guard for the post-loop. After leaving the
689 // main-loop, the post-loop may not execute at all. We 'opaque' the incr
690 // (the main-loop trip-counter exit value) because we will be changing
691 // the exit value (via unrolling) so we cannot constant-fold away the zero
692 // trip guard until all unrolling is done.
693 Node *zer_opaq = new (C, 2) Opaque1Node(C, incr);
694 Node *zer_cmp = new (C, 3) CmpINode( zer_opaq, limit );
695 Node *zer_bol = new (C, 2) BoolNode( zer_cmp, b_test );
696 register_new_node( zer_opaq, new_main_exit );
697 register_new_node( zer_cmp , new_main_exit );
698 register_new_node( zer_bol , new_main_exit );
699
700 // Build the IfNode
701 IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
731 _igvn.hash_delete(post_phi);
732 post_phi->set_req( LoopNode::EntryControl, fallmain );
733 }
734 }
735
736 // Update local caches for next stanza
737 main_exit = new_main_exit;
738
739
740 //------------------------------
741 // Step B: Create Pre-Loop.
742
743 // Step B1: Clone the loop body. The clone becomes the pre-loop. The main
744 // loop pre-header illegally has 2 control users (old & new loops).
745 clone_loop( loop, old_new, dd_main_head );
746 CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop();
747 CountedLoopEndNode* pre_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
748 pre_head->set_pre_loop(main_head);
749 Node *pre_incr = old_new[incr->_idx];
750
751 // Find the pre-loop normal exit.
752 Node* pre_exit = pre_end->proj_out(false);
753 assert( pre_exit->Opcode() == Op_IfFalse, "" );
754 IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
755 _igvn.register_new_node_with_optimizer( new_pre_exit );
756 set_idom(new_pre_exit, pre_end, dd_main_head);
757 set_loop(new_pre_exit, loop->_parent);
758
759 // Step B2: Build a zero-trip guard for the main-loop. After leaving the
760 // pre-loop, the main-loop may not execute at all. Later in life this
761 // zero-trip guard will become the minimum-trip guard when we unroll
762 // the main-loop.
763 Node *min_opaq = new (C, 2) Opaque1Node(C, limit);
764 Node *min_cmp = new (C, 3) CmpINode( pre_incr, min_opaq );
765 Node *min_bol = new (C, 2) BoolNode( min_cmp, b_test );
766 register_new_node( min_opaq, new_pre_exit );
767 register_new_node( min_cmp , new_pre_exit );
768 register_new_node( min_bol , new_pre_exit );
769
770 // Build the IfNode
771 IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_FAIR, COUNT_UNKNOWN );
772 _igvn.register_new_node_with_optimizer( min_iff );
773 set_idom(min_iff, new_pre_exit, dd_main_head);
774 set_loop(min_iff, loop->_parent);
775
776 // Plug in the false-path, taken if we need to skip main-loop
777 _igvn.hash_delete( pre_exit );
778 pre_exit->set_req(0, min_iff);
779 set_idom(pre_exit, min_iff, dd_main_head);
780 set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
781 // Make the true-path, must enter the main loop
782 Node *min_taken = new (C, 1) IfTrueNode( min_iff );
783 _igvn.register_new_node_with_optimizer( min_taken );
784 set_idom(min_taken, min_iff, dd_main_head);
785 set_loop(min_taken, loop->_parent);
786 // Plug in the true path
787 _igvn.hash_delete( main_head );
788 main_head->set_req(LoopNode::EntryControl, min_taken);
789 set_idom(main_head, min_taken, dd_main_head);
790
791 // Step B3: Make the fall-in values to the main-loop come from the
|
662 if( cmp->outcnt() != 1 ) {
663 cmp = cmp->clone();
664 register_new_node(cmp,main_end->in(CountedLoopEndNode::TestControl));
665 _igvn.hash_delete(bol);
666 bol->set_req(1, cmp);
667 }
668
669 //------------------------------
670 // Step A: Create Post-Loop.
671 Node* main_exit = main_end->proj_out(false);
672 assert( main_exit->Opcode() == Op_IfFalse, "" );
673 int dd_main_exit = dom_depth(main_exit);
674
675 // Step A1: Clone the loop body. The clone becomes the post-loop. The main
676 // loop pre-header illegally has 2 control users (old & new loops).
677 clone_loop( loop, old_new, dd_main_exit );
678 assert( old_new[main_end ->_idx]->Opcode() == Op_CountedLoopEnd, "" );
679 CountedLoopNode *post_head = old_new[main_head->_idx]->as_CountedLoop();
680 post_head->set_post_loop(main_head);
681
682 // Reduce the post-loop trip count.
683 CountedLoopEndNode* post_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
684 post_end->_prob = PROB_FAIR;
685
686 // Build the main-loop normal exit.
687 IfFalseNode *new_main_exit = new (C, 1) IfFalseNode(main_end);
688 _igvn.register_new_node_with_optimizer( new_main_exit );
689 set_idom(new_main_exit, main_end, dd_main_exit );
690 set_loop(new_main_exit, loop->_parent);
691
692 // Step A2: Build a zero-trip guard for the post-loop. After leaving the
693 // main-loop, the post-loop may not execute at all. We 'opaque' the incr
694 // (the main-loop trip-counter exit value) because we will be changing
695 // the exit value (via unrolling) so we cannot constant-fold away the zero
696 // trip guard until all unrolling is done.
697 Node *zer_opaq = new (C, 2) Opaque1Node(C, incr);
698 Node *zer_cmp = new (C, 3) CmpINode( zer_opaq, limit );
699 Node *zer_bol = new (C, 2) BoolNode( zer_cmp, b_test );
700 register_new_node( zer_opaq, new_main_exit );
701 register_new_node( zer_cmp , new_main_exit );
702 register_new_node( zer_bol , new_main_exit );
703
704 // Build the IfNode
705 IfNode *zer_iff = new (C, 2) IfNode( new_main_exit, zer_bol, PROB_FAIR, COUNT_UNKNOWN );
735 _igvn.hash_delete(post_phi);
736 post_phi->set_req( LoopNode::EntryControl, fallmain );
737 }
738 }
739
740 // Update local caches for next stanza
741 main_exit = new_main_exit;
742
743
744 //------------------------------
745 // Step B: Create Pre-Loop.
746
747 // Step B1: Clone the loop body. The clone becomes the pre-loop. The main
748 // loop pre-header illegally has 2 control users (old & new loops).
749 clone_loop( loop, old_new, dd_main_head );
750 CountedLoopNode* pre_head = old_new[main_head->_idx]->as_CountedLoop();
751 CountedLoopEndNode* pre_end = old_new[main_end ->_idx]->as_CountedLoopEnd();
752 pre_head->set_pre_loop(main_head);
753 Node *pre_incr = old_new[incr->_idx];
754
755 // Reduce the pre-loop trip count.
756 pre_end->_prob = PROB_FAIR;
757
758 // Find the pre-loop normal exit.
759 Node* pre_exit = pre_end->proj_out(false);
760 assert( pre_exit->Opcode() == Op_IfFalse, "" );
761 IfFalseNode *new_pre_exit = new (C, 1) IfFalseNode(pre_end);
762 _igvn.register_new_node_with_optimizer( new_pre_exit );
763 set_idom(new_pre_exit, pre_end, dd_main_head);
764 set_loop(new_pre_exit, loop->_parent);
765
766 // Step B2: Build a zero-trip guard for the main-loop. After leaving the
767 // pre-loop, the main-loop may not execute at all. Later in life this
768 // zero-trip guard will become the minimum-trip guard when we unroll
769 // the main-loop.
770 Node *min_opaq = new (C, 2) Opaque1Node(C, limit);
771 Node *min_cmp = new (C, 3) CmpINode( pre_incr, min_opaq );
772 Node *min_bol = new (C, 2) BoolNode( min_cmp, b_test );
773 register_new_node( min_opaq, new_pre_exit );
774 register_new_node( min_cmp , new_pre_exit );
775 register_new_node( min_bol , new_pre_exit );
776
777 // Build the IfNode (assume the main-loop is executed always).
778 IfNode *min_iff = new (C, 2) IfNode( new_pre_exit, min_bol, PROB_ALWAYS, COUNT_UNKNOWN );
779 _igvn.register_new_node_with_optimizer( min_iff );
780 set_idom(min_iff, new_pre_exit, dd_main_head);
781 set_loop(min_iff, loop->_parent);
782
783 // Plug in the false-path, taken if we need to skip main-loop
784 _igvn.hash_delete( pre_exit );
785 pre_exit->set_req(0, min_iff);
786 set_idom(pre_exit, min_iff, dd_main_head);
787 set_idom(pre_exit->unique_out(), min_iff, dd_main_head);
788 // Make the true-path, must enter the main loop
789 Node *min_taken = new (C, 1) IfTrueNode( min_iff );
790 _igvn.register_new_node_with_optimizer( min_taken );
791 set_idom(min_taken, min_iff, dd_main_head);
792 set_loop(min_taken, loop->_parent);
793 // Plug in the true path
794 _igvn.hash_delete( main_head );
795 main_head->set_req(LoopNode::EntryControl, min_taken);
796 set_idom(main_head, min_taken, dd_main_head);
797
798 // Step B3: Make the fall-in values to the main-loop come from the
|