x86: ignore high register select bit(s) in 32- and 16-bit modes

While commits 9889cbb14e ("Check invalid mask registers") and
abfcb414b9 ("X86: Ignore REX_B bit for 32-bit XOP instructions") went a
bit into the right direction, this wasn't quite enough:
- VEX.vvvv has its high bit ignored
- EVEX.vvvv has its high bit ignored together with EVEX.v'
- the high bits of {,E}VEX.vvvv should not be prematurely zapped, to
  allow proper checking of them when the fields has to hold al ones
- when the high bits of an immediate specify a register, bit 7 is
  ignored

2 files changed, 76 insertions, 4 deletions

diff --git a/gas/testsuite/gas/i386/noextreg.d b/gas/testsuite/gas/i386/noextreg.d
index 4b7f45d46e..99579c9c67 100644
--- a/gas/testsuite/gas/i386/noextreg.d
+++ b/gas/testsuite/gas/i386/noextreg.d
@@ -6,13 +6,48 @@
 Disassembly of section .text:
 
 0+ <ix86>:
+[ 	]*[a-f0-9]+:	c5 f9 db c0          	vpand  %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 c1 79 db c0       	vpand  %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 c1 39 db c0       	vpand  %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	62 f1 7d 08 db c0    	vpandd %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	62 d1 7d 08 db c0    	vpandd %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	62 f1 3d 08 db c0    	vpandd %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	62 f1 7d 00 db c0    	vpandd %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 79 4c c0 00    	vpblendvb %xmm0,%xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 c3 79 4c c0 00    	vpblendvb %xmm0,%xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 39 4c c0 00    	vpblendvb %xmm0,%xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 79 4c c0 80    	vpblendvb %xmm0,%xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	62 f2 7d 0f 90 0c 00 	vpgatherdd \(%eax,%xmm0,1\),%xmm1\{%k7\}
+[ 	]*[a-f0-9]+:	62 d2 7d 0f 90 0c 00 	vpgatherdd \(%eax,%xmm0,1\),%xmm1\{%k7\}
+[ 	]*[a-f0-9]+:	62 f2 7d 07 90 0c 00 	vpgatherdd \(%eax,%xmm0,1\),%xmm1\{%k7\}
 [ 	]*[a-f0-9]+:	c4 e2 78 f2 00       	andn   \(%eax\),%eax,%eax
+[ 	]*[a-f0-9]+:	c4 e2 38 f2 00       	andn   \(%eax\),%eax,%eax
+[ 	]*[a-f0-9]+:	c4 c2 78 f2 00       	andn   \(%eax\),%eax,%eax
 [ 	]*[a-f0-9]+:	c4 e2 f8 f2 00       	andn   \(%eax\),%eax,%eax
 [ 	]*[a-f0-9]+:	8f e9 78 01 20       	tzmsk  \(%eax\),%eax
+[ 	]*[a-f0-9]+:	8f c9 78 01 20       	tzmsk  \(%eax\),%eax
+[ 	]*[a-f0-9]+:	8f e9 38 01 20       	tzmsk  \(%eax\),%eax
 [ 	]*[a-f0-9]+:	8f e9 f8 01 20       	tzmsk  \(%eax\),%eax
 [ 	]*[a-f0-9]+:	8f e9 78 12 c0       	llwpcb %eax
+[ 	]*[a-f0-9]+:	8f c9 78 12 c0       	llwpcb %eax
 [ 	]*[a-f0-9]+:	8f e9 f8 12 c0       	llwpcb %eax
+[ 	]*[a-f0-9]+:	8f e8 78 c0 c0 01    	vprotb \$0x1,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	8f c8 78 c0 c0 01    	vprotb \$0x1,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	8f e8 78 c0 00 01    	vprotb \$0x1,\(%eax\),%xmm0
+[ 	]*[a-f0-9]+:	8f c8 78 c0 00 01    	vprotb \$0x1,\(%eax\),%xmm0
+[ 	]*[a-f0-9]+:	8f e9 78 90 c0       	vprotb %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	8f c9 b8 90 c0       	vprotb %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	8f e9 38 90 c0       	vprotb %xmm0,%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	8f e9 78 90 00       	vprotb %xmm0,\(%eax\),%xmm0
+[ 	]*[a-f0-9]+:	8f c9 78 90 00       	vprotb %xmm0,\(%eax\),%xmm0
+[ 	]*[a-f0-9]+:	8f e9 f8 90 00       	vprotb \(%eax\),%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	8f c9 f8 90 00       	vprotb \(%eax\),%xmm0,%xmm0
 [ 	]*[a-f0-9]+:	c4 e3 79 68 00 00    	vfmaddps %xmm0,\(%eax\),%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 39 68 00 00    	vfmaddps %xmm0,\(%eax\),%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 79 68 00 80    	vfmaddps %xmm0,\(%eax\),%xmm0,%xmm0
 [ 	]*[a-f0-9]+:	c4 e3 79 68 00 0f    	vfmaddps %xmm0,\(%eax\),%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 79 48 00 00    	vpermil2ps \$0x0,%xmm0,\(%eax\),%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 39 48 00 00    	vpermil2ps \$0x0,%xmm0,\(%eax\),%xmm0,%xmm0
+[ 	]*[a-f0-9]+:	c4 e3 79 48 00 80    	vpermil2ps \$0x0,%xmm0,\(%eax\),%xmm0,%xmm0
 [ 	]*[a-f0-9]+:	c3                   	ret[ 	]*
 #pass
diff --git a/gas/testsuite/gas/i386/noextreg.s b/gas/testsuite/gas/i386/noextreg.s
index f1205acf22..85fd26ed5a 100644
--- a/gas/testsuite/gas/i386/noextreg.s
+++ b/gas/testsuite/gas/i386/noextreg.s
@@ -1,20 +1,57 @@
 	.intel_syntax noprefix
 	.text
 ix86:
+	vpand	xmm0, xmm0, xmm0
+	.byte	0xc4, 0xc1, 0x79, 0xdb, 0xc0
+	.byte	0xc4, 0xc1, 0x39, 0xdb, 0xc0
+
+	vpandd	xmm0, xmm0, xmm0
+	.byte	0x62, 0xd1, 0x7d, 0x08, 0xdb, 0xc0
+	.byte	0x62, 0xf1, 0x3d, 0x08, 0xdb, 0xc0
+	.byte	0x62, 0xf1, 0x7d, 0x00, 0xdb, 0xc0
+
+	vpblendvb xmm0, xmm0, xmm0, xmm0
+	.byte	0xc4, 0xc3, 0x79, 0x4c, 0xc0, 0x00
+	.byte	0xc4, 0xe3, 0x39, 0x4c, 0xc0, 0x00
+	.byte	0xc4, 0xe3, 0x79, 0x4c, 0xc0, 0x80
+
+	vpgatherdd xmm1{k7}, [eax+xmm0]
+	.byte	0x62, 0xd2, 0x7d, 0x0f, 0x90, 0x0c, 0x00
+	.byte	0x62, 0xf2, 0x7d, 0x07, 0x90, 0x0c, 0x00
+
 	andn	eax, eax, [eax]
-	# andn rax, rax, [rax]
+	.byte	0xc4, 0xe2, 0x38, 0xf2, 0x00
+	.byte	0xc4, 0xc2, 0x78, 0xf2, 0x00
 	.byte	0xc4, 0xe2, 0xf8, 0xf2, 0x00
 
 	tzmsk	eax, [eax]
-	# tzmsk rax, [rax]
+	.byte	0x8f, 0xc9, 0x78, 0x01, 0x20
+	.byte	0x8f, 0xe9, 0x38, 0x01, 0x20
 	.byte	0x8f, 0xe9, 0xf8, 0x01, 0x20
 
 	llwpcb	eax
-	# llwpcb rax
+	.byte	0x8f, 0xc9, 0x78, 0x12, 0xc0
 	.byte	0x8f, 0xe9, 0xf8, 0x12, 0xc0
 
+	vprotb	xmm0, xmm0, 1
+	.byte	0x8f, 0xc8, 0x78, 0xc0, 0xc0, 0x01
+	vprotb	xmm0, [eax], 1
+	.byte	0x8f, 0xc8, 0x78, 0xc0, 0x00, 0x01
+	vprotb	xmm0, xmm0, xmm0
+	.byte	0x8f, 0xc9, 0xb8, 0x90, 0xc0
+	.byte	0x8f, 0xe9, 0x38, 0x90, 0xc0
+	vprotb	xmm0, [eax], xmm0
+	.byte	0x8f, 0xc9, 0x78, 0x90, 0x00
+	vprotb	xmm0, xmm0, [eax]
+	.byte	0x8f, 0xc9, 0xf8, 0x90, 0x00
+
 	vfmaddps xmm0, xmm0, [eax], xmm0
-	# vfmaddps xmm0, xmm0, [eax], xmm0
+	.byte	0xc4, 0xe3, 0x39, 0x68, 0x00, 0x00
+	.byte	0xc4, 0xe3, 0x79, 0x68, 0x00, 0x80
 	.byte	0xc4, 0xe3, 0x79, 0x68, 0x00, 0x0f
 
+	vpermil2ps xmm0, xmm0, [eax], xmm0, 0
+	.byte	0xc4, 0xe3, 0x39, 0x48, 0x00, 0x00
+	.byte	0xc4, 0xe3, 0x79, 0x48, 0x00, 0x80
+
 	ret