x86: ignore high register select bit(s) in 32- and 16-bit modes

While commits 9889cbb14e ("Check invalid mask registers") and
abfcb414b9 ("X86: Ignore REX_B bit for 32-bit XOP instructions") went a
bit into the right direction, this wasn't quite enough:
- VEX.vvvv has its high bit ignored
- EVEX.vvvv has its high bit ignored together with EVEX.v'
- the high bits of {,E}VEX.vvvv should not be prematurely zapped, to
  allow proper checking of them when the fields has to hold al ones
- when the high bits of an immediate specify a register, bit 7 is
  ignored

2 files changed, 47 insertions, 28 deletions

diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index 2cc35947bd..6ea9c100bf 100644
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,3 +1,14 @@
+2017-11-16  Jan Beulich  <jbeulich@suse.com>
+
+	(get_valid_dis386): Never flag bad opcode when
+	vex.register_specifier is beyond 7. Always store all four
+	bits of it. Move 16-/32-bit override in EVEX handling after
+	all to be overridden bits have been set.
+	(OP_VEX): Mask vex.register_specifier outside of 64-bit mode.
+	Use rex to determine GPR register set.
+	(OP_EX_VexReg, OP_Vex_2src_1, OP_Vex_2src_2, OP_REG_VexI4,
+	OP_LWP_E): Mask vex.register_specifier outside of 64-bit mode.
+
 2017-11-15  Jan Beulich  <jbeulich@suse.com>
 
 	* i386-dis.c (OP_VEX, OP_LWPCB_E, OP_LWP_E): Use rex to
diff --git a/opcodes/i386-dis.c b/opcodes/i386-dis.c
index 78a685ee47..64b47028fd 100644
--- a/opcodes/i386-dis.c
+++ b/opcodes/i386-dis.c
@@ -12809,11 +12809,6 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
 	{
 	  /* In 16/32-bit mode REX_B is silently ignored.  */
 	  rex &= ~REX_B;
-	  if (vex.register_specifier > 0x7)
-	    {
-	      dp = &bad_opcode;
-	      return dp;
-	    }
 	}
 
       vex.length = (*codep & 0x4) ? 256 : 128;
@@ -12872,16 +12867,15 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
 	{
 	  if (vex.w)
 	    rex |= REX_W;
-	  vex.register_specifier = (~(*codep >> 3)) & 0xf;
 	}
       else
 	{
 	  /* For the 3-byte VEX prefix in 32-bit mode, the REX_B bit
 	     is ignored, other REX bits are 0 and the highest bit in
-	     VEX.vvvv is also ignored.  */
+	     VEX.vvvv is also ignored (but we mustn't clear it here).  */
 	  rex = 0;
-	  vex.register_specifier = (~(*codep >> 3)) & 0x7;
 	}
+      vex.register_specifier = (~(*codep >> 3)) & 0xf;
       vex.length = (*codep & 0x4) ? 256 : 128;
       switch ((*codep & 0x3))
 	{
@@ -12996,14 +12990,6 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
 	rex |= REX_W;
 
       vex.register_specifier = (~(*codep >> 3)) & 0xf;
-      if (address_mode != mode_64bit)
-	{
-	  /* In 16/32-bit mode silently ignore following bits.  */
-	  rex &= ~REX_B;
-	  vex.r = 1;
-	  vex.v = 1;
-	  vex.register_specifier &= 0x7;
-	}
 
       /* The U bit.  */
       if (!(*codep & 0x4))
@@ -13036,6 +13022,14 @@ get_valid_dis386 (const struct dis386 *dp, disassemble_info *info)
       vex.mask_register_specifier = *codep & 0x7;
       vex.zeroing = *codep & 0x80;
 
+      if (address_mode != mode_64bit)
+	{
+	  /* In 16/32-bit mode silently ignore following bits.  */
+	  rex &= ~REX_B;
+	  vex.r = 1;
+	  vex.v = 1;
+	}
+
       need_vex = 1;
       need_vex_reg = 1;
       codep++;
@@ -17098,11 +17092,10 @@ OP_VEX (int bytemode, int sizeflag ATTRIBUTE_UNUSED)
     return;
 
   reg = vex.register_specifier;
-  if (vex.evex)
-    {
-      if (!vex.v)
-	reg += 16;
-    }
+  if (address_mode != mode_64bit)
+    reg &= 7;
+  else if (vex.evex && !vex.v)
+    reg += 16;
 
   if (bytemode == vex_scalar_mode)
     {
@@ -17287,8 +17280,8 @@ OP_EX_VexReg (int bytemode, int sizeflag, int reg)
 	  if (rex & REX_B)
 	    reg += 8;
 	}
-      else if (reg > 7 && address_mode != mode_64bit)
-	BadOp ();
+      if (address_mode != mode_64bit)
+	reg &= 7;
     }
 
   switch (vex.length)
@@ -17380,7 +17373,13 @@ OP_Vex_2src_1 (int bytemode, int sizeflag)
     }
 
   if (vex.w)
-    oappend (names_xmm[vex.register_specifier]);
+    {
+      unsigned int reg = vex.register_specifier;
+
+      if (address_mode != mode_64bit)
+	reg &= 7;
+      oappend (names_xmm[reg]);
+    }
   else
     OP_Vex_2src (bytemode, sizeflag);
 }
@@ -17391,7 +17390,13 @@ OP_Vex_2src_2 (int bytemode, int sizeflag)
   if (vex.w)
     OP_Vex_2src (bytemode, sizeflag);
   else
-    oappend (names_xmm[vex.register_specifier]);
+    {
+      unsigned int reg = vex.register_specifier;
+
+      if (address_mode != mode_64bit)
+	reg &= 7;
+      oappend (names_xmm[reg]);
+    }
 }
 
 static void
@@ -17434,8 +17439,8 @@ OP_REG_VexI4 (int bytemode, int sizeflag ATTRIBUTE_UNUSED)
     abort ();
 
   reg >>= 4;
-  if (reg > 7 && address_mode != mode_64bit)
-    BadOp ();
+  if (address_mode != mode_64bit)
+    reg &= 7;
 
   switch (vex.length)
     {
@@ -17775,13 +17780,16 @@ static void
 OP_LWP_E (int bytemode ATTRIBUTE_UNUSED, int sizeflag ATTRIBUTE_UNUSED)
 {
   const char **names;
+  unsigned int reg = vex.register_specifier;
 
   if (rex & REX_W)
     names = names64;
   else
     names = names32;
 
-  oappend (names[vex.register_specifier]);
+  if (address_mode != mode_64bit)
+    reg &= 7;
+  oappend (names[reg]);
 }
 
 static void