x86: Disallow -mindirect-branch=/-mfunction-return= with -mcmodel=large

Since the thunk function may not be reachable in large code model,
-mcmodel=large is incompatible with -mindirect-branch=thunk,
-mindirect-branch=thunk-extern, -mfunction-return=thunk and
-mfunction-return=thunk-extern.  Issue an error when they are used with
-mcmodel=large.

gcc/

	Backport from mainline
	2018-01-14  H.J. Lu  <hongjiu.lu@intel.com>

	* config/i386/i386.c (ix86_set_indirect_branch_type): Disallow
	-mcmodel=large with -mindirect-branch=thunk,
	-mindirect-branch=thunk-extern, -mfunction-return=thunk and
	-mfunction-return=thunk-extern.
	* doc/invoke.texi: Document -mcmodel=large is incompatible with
	-mindirect-branch=thunk, -mindirect-branch=thunk-extern,
	-mfunction-return=thunk and -mfunction-return=thunk-extern.

gcc/testsuite/

	Backport from mainline
	2018-01-14  H.J. Lu  <hongjiu.lu@intel.com>

	* gcc.target/i386/indirect-thunk-10.c: New test.
	* gcc.target/i386/indirect-thunk-8.c: Likewise.
	* gcc.target/i386/indirect-thunk-9.c: Likewise.
	* gcc.target/i386/indirect-thunk-attr-10.c: Likewise.
	* gcc.target/i386/indirect-thunk-attr-11.c: Likewise.
	* gcc.target/i386/indirect-thunk-attr-9.c: Likewise.
	* gcc.target/i386/ret-thunk-17.c: Likewise.
	* gcc.target/i386/ret-thunk-18.c: Likewise.
	* gcc.target/i386/ret-thunk-19.c: Likewise.
	* gcc.target/i386/ret-thunk-20.c: Likewise.
	* gcc.target/i386/ret-thunk-21.c: Likewise.


git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/branches/gcc-7-branch@256737 138bc75d-0d04-0410-961f-82ee72b054a4

15 files changed, 156 insertions, 0 deletions

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index fc222f064a92..1ef52541038d 100644
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -3,6 +3,19 @@
 	Backport from mainline
 	2018-01-14  H.J. Lu  <hongjiu.lu@intel.com>
 
+	* config/i386/i386.c (ix86_set_indirect_branch_type): Disallow
+	-mcmodel=large with -mindirect-branch=thunk,
+	-mindirect-branch=thunk-extern, -mfunction-return=thunk and
+	-mfunction-return=thunk-extern.
+	* doc/invoke.texi: Document -mcmodel=large is incompatible with
+	-mindirect-branch=thunk, -mindirect-branch=thunk-extern,
+	-mfunction-return=thunk and -mfunction-return=thunk-extern.
+
+2018-01-16  H.J. Lu  <hongjiu.lu@intel.com>
+
+	Backport from mainline
+	2018-01-14  H.J. Lu  <hongjiu.lu@intel.com>
+
 	* config/i386/i386.c (print_reg): Print the name of the full
 	integer register without '%'.
 	(ix86_print_operand): Handle 'V'.
diff --git a/gcc/config/i386/i386.c b/gcc/config/i386/i386.c
index 1bbdd0cc3f8c..e758387dacd0 100644
--- a/gcc/config/i386/i386.c
+++ b/gcc/config/i386/i386.c
@@ -7187,6 +7187,19 @@ ix86_set_indirect_branch_type (tree fndecl)
 	}
       else
 	cfun->machine->indirect_branch_type = ix86_indirect_branch;
+
+      /* -mcmodel=large is not compatible with -mindirect-branch=thunk
+	 nor -mindirect-branch=thunk-extern.  */
+      if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
+	  && ((cfun->machine->indirect_branch_type
+	       == indirect_branch_thunk_extern)
+	      || (cfun->machine->indirect_branch_type
+		  == indirect_branch_thunk)))
+	error ("%<-mindirect-branch=%s%> and %<-mcmodel=large%> are not "
+	       "compatible",
+	       ((cfun->machine->indirect_branch_type
+		 == indirect_branch_thunk_extern)
+		? "thunk-extern" : "thunk"));
     }
 
   if (cfun->machine->function_return_type == indirect_branch_unset)
@@ -7212,6 +7225,19 @@ ix86_set_indirect_branch_type (tree fndecl)
 	}
       else
 	cfun->machine->function_return_type = ix86_function_return;
+
+      /* -mcmodel=large is not compatible with -mfunction-return=thunk
+	 nor -mfunction-return=thunk-extern.  */
+      if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC)
+	  && ((cfun->machine->function_return_type
+	       == indirect_branch_thunk_extern)
+	      || (cfun->machine->function_return_type
+		  == indirect_branch_thunk)))
+	error ("%<-mfunction-return=%s%> and %<-mcmodel=large%> are not "
+	       "compatible",
+	       ((cfun->machine->function_return_type
+		 == indirect_branch_thunk_extern)
+		? "thunk-extern" : "thunk"));
     }
 }
 
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 1e572b1f9a25..6f3c344476c6 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -25699,6 +25699,11 @@ to external call and return thunk provided in a separate object file.
 You can control this behavior for a specific function by using the
 function attribute @code{indirect_branch}.  @xref{Function Attributes}.
 
+Note that @option{-mcmodel=large} is incompatible with
+@option{-mindirect-branch=thunk} nor
+@option{-mindirect-branch=thunk-extern} since the thunk function may
+not be reachable in large code model.
+
 @item -mfunction-return=@var{choice}
 @opindex -mfunction-return
 Convert function return with @var{choice}.  The default is @samp{keep},
@@ -25710,6 +25715,12 @@ object file.  You can control this behavior for a specific function by
 using the function attribute @code{function_return}.
 @xref{Function Attributes}.
 
+Note that @option{-mcmodel=large} is incompatible with
+@option{-mfunction-return=thunk} nor
+@option{-mfunction-return=thunk-extern} since the thunk function may
+not be reachable in large code model.
+
+
 @item -mindirect-branch-register
 @opindex -mindirect-branch-register
 Force indirect call and jump via register.
diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog
index b2e85383dd18..6752d761d844 100644
--- a/gcc/testsuite/ChangeLog
+++ b/gcc/testsuite/ChangeLog
@@ -3,6 +3,23 @@
 	Backport from mainline
 	2018-01-14  H.J. Lu  <hongjiu.lu@intel.com>
 
+	* gcc.target/i386/indirect-thunk-10.c: New test.
+	* gcc.target/i386/indirect-thunk-8.c: Likewise.
+	* gcc.target/i386/indirect-thunk-9.c: Likewise.
+	* gcc.target/i386/indirect-thunk-attr-10.c: Likewise.
+	* gcc.target/i386/indirect-thunk-attr-11.c: Likewise.
+	* gcc.target/i386/indirect-thunk-attr-9.c: Likewise.
+	* gcc.target/i386/ret-thunk-17.c: Likewise.
+	* gcc.target/i386/ret-thunk-18.c: Likewise.
+	* gcc.target/i386/ret-thunk-19.c: Likewise.
+	* gcc.target/i386/ret-thunk-20.c: Likewise.
+	* gcc.target/i386/ret-thunk-21.c: Likewise.
+
+2018-01-16  H.J. Lu  <hongjiu.lu@intel.com>
+
+	Backport from mainline
+	2018-01-14  H.J. Lu  <hongjiu.lu@intel.com>
+
 	* gcc.target/i386/indirect-thunk-register-4.c: New test.
 
 2018-01-16  H.J. Lu  <hongjiu.lu@intel.com>
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-10.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-10.c
new file mode 100644
index 000000000000..a0674bd23639
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-10.c
@@ -0,0 +1,7 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mindirect-branch=thunk-inline -mfunction-return=keep -mcmodel=large" } */
+
+void
+bar (void)
+{
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-8.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-8.c
new file mode 100644
index 000000000000..7a80a8986e87
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-8.c
@@ -0,0 +1,7 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mindirect-branch=thunk -mfunction-return=keep -mcmodel=large" } */
+
+void
+bar (void)
+{ /* { dg-error "'-mindirect-branch=thunk' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-9.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-9.c
new file mode 100644
index 000000000000..d4d45c5114dd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-9.c
@@ -0,0 +1,7 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mindirect-branch=thunk-extern -mfunction-return=keep -mcmodel=large" } */
+
+void
+bar (void)
+{ /* { dg-error "'-mindirect-branch=thunk-extern' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-10.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-10.c
new file mode 100644
index 000000000000..3a2aeaddbc5c
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-10.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mindirect-branch=keep -mfunction-return=keep -mcmodel=large" } */
+/* { dg-additional-options "-fPIC" { target fpic } } */
+
+__attribute__ ((indirect_branch("thunk-extern")))
+void
+bar (void)
+{ /* { dg-error "'-mindirect-branch=thunk-extern' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-11.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-11.c
new file mode 100644
index 000000000000..8e52f032b6c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-11.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mindirect-branch=keep -mfunction-return=keep -mcmodel=large" } */
+/* { dg-additional-options "-fPIC" { target fpic } } */
+
+__attribute__ ((indirect_branch("thunk-inline")))
+void
+bar (void)
+{
+}
diff --git a/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-9.c b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-9.c
new file mode 100644
index 000000000000..bdaa4f6911bd
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/indirect-thunk-attr-9.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mindirect-branch=keep -mfunction-return=keep -mcmodel=large" } */
+/* { dg-additional-options "-fPIC" { target fpic } } */
+
+__attribute__ ((indirect_branch("thunk")))
+void
+bar (void)
+{ /* { dg-error "'-mindirect-branch=thunk' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-17.c b/gcc/testsuite/gcc.target/i386/ret-thunk-17.c
new file mode 100644
index 000000000000..0605e2c65424
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-17.c
@@ -0,0 +1,7 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mfunction-return=thunk -mindirect-branch=keep -mcmodel=large" } */
+
+void
+bar (void)
+{ /* { dg-error "'-mfunction-return=thunk' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-18.c b/gcc/testsuite/gcc.target/i386/ret-thunk-18.c
new file mode 100644
index 000000000000..307019dc2424
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-18.c
@@ -0,0 +1,8 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mfunction-return=thunk-extern -mindirect-branch=keep -mcmodel=large" } */
+/* { dg-additional-options "-fPIC" { target fpic } } */
+
+void
+bar (void)
+{ /* { dg-error "'-mfunction-return=thunk-extern' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-19.c b/gcc/testsuite/gcc.target/i386/ret-thunk-19.c
new file mode 100644
index 000000000000..772617f40102
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-19.c
@@ -0,0 +1,8 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mfunction-return=keep -mindirect-branch=keep -mcmodel=large" } */
+
+__attribute__ ((function_return("thunk")))
+void
+bar (void)
+{ /* { dg-error "'-mfunction-return=thunk' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-20.c b/gcc/testsuite/gcc.target/i386/ret-thunk-20.c
new file mode 100644
index 000000000000..1e9f9bd5a661
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-20.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mfunction-return=keep -mindirect-branch=keep -mcmodel=large" } */
+/* { dg-additional-options "-fPIC" { target fpic } } */
+
+__attribute__ ((function_return("thunk-extern")))
+void
+bar (void)
+{ /* { dg-error "'-mfunction-return=thunk-extern' and '-mcmodel=large' are not compatible" } */
+}
diff --git a/gcc/testsuite/gcc.target/i386/ret-thunk-21.c b/gcc/testsuite/gcc.target/i386/ret-thunk-21.c
new file mode 100644
index 000000000000..eea07f7abe11
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/ret-thunk-21.c
@@ -0,0 +1,9 @@
+/* { dg-do compile { target { lp64 } } } */
+/* { dg-options "-O2 -mfunction-return=keep -mindirect-branch=keep -mcmodel=large" } */
+/* { dg-additional-options "-fPIC" { target fpic } } */
+
+__attribute__ ((function_return("thunk-inline")))
+void
+bar (void)
+{
+}