diff options
author | Yonghong Song <yhs@fb.com> | 2017-06-16 15:41:16 +0000 |
---|---|---|
committer | Yonghong Song <yhs@fb.com> | 2017-06-16 15:41:16 +0000 |
commit | 1aa4ba7ed969272250c7647ff14305f5b2f32c26 (patch) | |
tree | 3cc05ddf39c7437530970f3dd24280a5abc97362 | |
parent | 211587773d3a4438cb229157189de58c482822d1 (diff) |
bpf: avoid load from read-only sections
If users tried to have a structure decl/init code like below
struct test_t t = { .memeber1 = 45 };
It is very likely that compiler will generate a readonly section
to hold up the init values for variable t. Later load of t members,
e.g., t.member1 will result in a read from readonly section.
BPF program cannot handle relocation. This will force users to
write:
struct test_t t = {};
t.member1 = 45;
This is just inconvenient and unintuitive.
This patch addresses this issue by implementing BPF PreprocessISelDAG.
For any load from a global constant structure or an global array of
constant struct, it attempts to
translate it into a constant directly. The traversal of the
constant struct and other constant data structures are similar
to where the assembler emits read-only sections.
Four different unit test cases are also added to cover
different scenarios.
Signed-off-by: Yonghong Song <yhs@fb.com>
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@305560 91177308-0d34-0410-b5e6-96231b3b80d8
-rw-r--r-- | lib/Target/BPF/BPFISelDAGToDAG.cpp | 240 | ||||
-rw-r--r-- | test/CodeGen/BPF/rodata_1.ll | 52 | ||||
-rw-r--r-- | test/CodeGen/BPF/rodata_2.ll | 51 | ||||
-rw-r--r-- | test/CodeGen/BPF/rodata_3.ll | 41 | ||||
-rw-r--r-- | test/CodeGen/BPF/rodata_4.ll | 43 |
5 files changed, 420 insertions, 7 deletions
diff --git a/lib/Target/BPF/BPFISelDAGToDAG.cpp b/lib/Target/BPF/BPFISelDAGToDAG.cpp index 279cdb1a89b..7d5fb6ca17b 100644 --- a/lib/Target/BPF/BPFISelDAGToDAG.cpp +++ b/lib/Target/BPF/BPFISelDAGToDAG.cpp @@ -22,11 +22,14 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/SelectionDAGISel.h" +#include "llvm/IR/Constants.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" + using namespace llvm; #define DEBUG_TYPE "bpf-isel" @@ -42,6 +45,8 @@ public: return "BPF DAG->DAG Pattern Instruction Selection"; } + void PreprocessISelDAG() override; + private: // Include the pieces autogenerated from the target description. #include "BPFGenDAGISel.inc" @@ -51,15 +56,31 @@ private: // Complex Pattern for address selection. bool SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset); bool SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset); + + // Find constants from a constant structure + typedef std::vector<unsigned char> val_vec_type; + bool fillGenericConstant(const DataLayout &DL, const Constant *CV, + val_vec_type &Vals, uint64_t Offset); + bool fillConstantDataArray(const DataLayout &DL, const ConstantDataArray *CDA, + val_vec_type &Vals, int Offset); + bool fillConstantArray(const DataLayout &DL, const ConstantArray *CA, + val_vec_type &Vals, int Offset); + bool fillConstantStruct(const DataLayout &DL, const ConstantStruct *CS, + val_vec_type &Vals, int Offset); + bool getConstantFieldValue(const GlobalAddressSDNode *Node, uint64_t Offset, + uint64_t Size, unsigned char *ByteSeq); + + // Mapping from ConstantStruct global value to corresponding byte-list values + std::map<const void *, val_vec_type> cs_vals_; }; -} +} // namespace // ComplexPattern used on BPF Load/Store instructions bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) { // if Address is FI, get the TargetFrameIndex. SDLoc DL(Addr); if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) { - Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); + Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); Offset = CurDAG->getTargetConstant(0, DL, MVT::i64); return true; } @@ -85,13 +106,14 @@ bool BPFDAGToDAGISel::SelectAddr(SDValue Addr, SDValue &Base, SDValue &Offset) { } } - Base = Addr; + Base = Addr; Offset = CurDAG->getTargetConstant(0, DL, MVT::i64); return true; } // ComplexPattern used on BPF FI instruction -bool BPFDAGToDAGISel::SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset) { +bool BPFDAGToDAGISel::SelectFIAddr(SDValue Addr, SDValue &Base, + SDValue &Offset) { SDLoc DL(Addr); if (!CurDAG->isBaseWithConstantOffset(Addr)) @@ -102,8 +124,7 @@ bool BPFDAGToDAGISel::SelectFIAddr(SDValue Addr, SDValue &Base, SDValue &Offset) if (isInt<16>(CN->getSExtValue())) { // If the first operand is a FI, get the TargetFI Node - if (FrameIndexSDNode *FIN = - dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) + if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), MVT::i64); else return false; @@ -129,7 +150,8 @@ void BPFDAGToDAGISel::Select(SDNode *Node) { // tablegen selection should be handled here. switch (Opcode) { - default: break; + default: + break; case ISD::SDIV: { DebugLoc Empty; const DebugLoc &DL = Node->getDebugLoc(); @@ -181,6 +203,210 @@ void BPFDAGToDAGISel::Select(SDNode *Node) { SelectCode(Node); } +void BPFDAGToDAGISel::PreprocessISelDAG() { + // Iterate through all nodes, only interested in loads from ConstantStruct + // ConstantArray should have converted by IR->DAG processing + for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), + E = CurDAG->allnodes_end(); + I != E;) { + SDNode *Node = &*I++; + unsigned Opcode = Node->getOpcode(); + if (Opcode != ISD::LOAD) + continue; + + unsigned char new_val[8]; // hold up the constant values replacing loads. + bool to_replace = false; + SDLoc DL(Node); + const LoadSDNode *LD = cast<LoadSDNode>(Node); + uint64_t size = LD->getMemOperand()->getSize(); + if (!size || size > 8 || (size & (size - 1))) + continue; + + SDNode *LDAddrNode = LD->getOperand(1).getNode(); + // Match LDAddr against either global_addr or (global_addr + offset) + unsigned opcode = LDAddrNode->getOpcode(); + if (opcode == ISD::ADD) { + SDValue OP1 = LDAddrNode->getOperand(0); + SDValue OP2 = LDAddrNode->getOperand(1); + + // We want to find the pattern global_addr + offset + SDNode *OP1N = OP1.getNode(); + if (OP1N->getOpcode() <= ISD::BUILTIN_OP_END || + OP1N->getNumOperands() == 0) + continue; + + DEBUG(dbgs() << "Check candidate load: "; LD->dump(); dbgs() << '\n'); + + const GlobalAddressSDNode *GADN = + dyn_cast<GlobalAddressSDNode>(OP1N->getOperand(0).getNode()); + const ConstantSDNode *CDN = dyn_cast<ConstantSDNode>(OP2.getNode()); + if (GADN && CDN) + to_replace = + getConstantFieldValue(GADN, CDN->getZExtValue(), size, new_val); + } else if (LDAddrNode->getOpcode() > ISD::BUILTIN_OP_END && + LDAddrNode->getNumOperands() > 0) { + DEBUG(dbgs() << "Check candidate load: "; LD->dump(); dbgs() << '\n'); + + SDValue OP1 = LDAddrNode->getOperand(0); + if (const GlobalAddressSDNode *GADN = + dyn_cast<GlobalAddressSDNode>(OP1.getNode())) + to_replace = getConstantFieldValue(GADN, 0, size, new_val); + } + + if (!to_replace) + continue; + + // replacing the old with a new value + uint64_t val; + if (size == 1) + val = *(uint8_t *)new_val; + else if (size == 2) + val = *(uint16_t *)new_val; + else if (size == 4) + val = *(uint32_t *)new_val; + else { + val = *(uint64_t *)new_val; + } + + DEBUG(dbgs() << "Replacing load of size " << size << " with constant " + << val << '\n'); + SDValue NVal = CurDAG->getConstant(val, DL, MVT::i64); + + // After replacement, the current node is dead, we need to + // go backward one step to make iterator still work + I--; + SDValue From[] = {SDValue(Node, 0), SDValue(Node, 1)}; + SDValue To[] = {NVal, NVal}; + CurDAG->ReplaceAllUsesOfValuesWith(From, To, 2); + I++; + // It is safe to delete node now + CurDAG->DeleteNode(Node); + } +} + +bool BPFDAGToDAGISel::getConstantFieldValue(const GlobalAddressSDNode *Node, + uint64_t Offset, uint64_t Size, + unsigned char *ByteSeq) { + const GlobalVariable *V = dyn_cast<GlobalVariable>(Node->getGlobal()); + + if (!V || !V->hasInitializer()) + return false; + + const Constant *Init = V->getInitializer(); + const DataLayout &DL = CurDAG->getDataLayout(); + val_vec_type TmpVal; + + auto it = cs_vals_.find(static_cast<const void *>(Init)); + if (it != cs_vals_.end()) { + TmpVal = it->second; + } else { + uint64_t total_size = 0; + if (const ConstantStruct *CS = dyn_cast<ConstantStruct>(Init)) + total_size = + DL.getStructLayout(cast<StructType>(CS->getType()))->getSizeInBytes(); + else if (const ConstantArray *CA = dyn_cast<ConstantArray>(Init)) + total_size = DL.getTypeAllocSize(CA->getType()->getElementType()) * + CA->getNumOperands(); + else + return false; + + val_vec_type Vals(total_size, 0); + if (fillGenericConstant(DL, Init, Vals, 0) == false) + return false; + cs_vals_[static_cast<const void *>(Init)] = Vals; + TmpVal = std::move(Vals); + } + + // test whether host endianness matches target + uint8_t test_buf[2]; + uint16_t test_val = 0x2345; + if (DL.isLittleEndian()) + support::endian::write16le(test_buf, test_val); + else + support::endian::write16be(test_buf, test_val); + + bool endian_match = *(uint16_t *)test_buf == test_val; + for (uint64_t i = Offset, j = 0; i < Offset + Size; i++, j++) + ByteSeq[j] = endian_match ? TmpVal[i] : TmpVal[Offset + Size - 1 - j]; + + return true; +} + +bool BPFDAGToDAGISel::fillGenericConstant(const DataLayout &DL, + const Constant *CV, + val_vec_type &Vals, uint64_t Offset) { + uint64_t Size = DL.getTypeAllocSize(CV->getType()); + + if (isa<ConstantAggregateZero>(CV) || isa<UndefValue>(CV)) + return true; // already done + + if (const ConstantInt *CI = dyn_cast<ConstantInt>(CV)) { + uint64_t val = CI->getZExtValue(); + DEBUG(dbgs() << "Byte array at offset " << Offset << " with value " << val + << '\n'); + + if (Size > 8 || (Size & (Size - 1))) + return false; + + // Store based on target endian + for (uint64_t i = 0; i < Size; ++i) { + Vals[Offset + i] = DL.isLittleEndian() + ? ((val >> (i * 8)) & 0xFF) + : ((val >> ((Size - i - 1) * 8)) & 0xFF); + } + return true; + } + + if (const ConstantDataArray *CDA = dyn_cast<ConstantDataArray>(CV)) + return fillConstantDataArray(DL, CDA, Vals, Offset); + + if (const ConstantArray *CA = dyn_cast<ConstantArray>(CV)) + return fillConstantArray(DL, CA, Vals, Offset); + + if (const ConstantStruct *CVS = dyn_cast<ConstantStruct>(CV)) + return fillConstantStruct(DL, CVS, Vals, Offset); + + return false; +} + +bool BPFDAGToDAGISel::fillConstantDataArray(const DataLayout &DL, + const ConstantDataArray *CDA, + val_vec_type &Vals, int Offset) { + for (unsigned i = 0, e = CDA->getNumElements(); i != e; ++i) { + if (fillGenericConstant(DL, CDA->getElementAsConstant(i), Vals, Offset) == + false) + return false; + Offset += DL.getTypeAllocSize(CDA->getElementAsConstant(i)->getType()); + } + + return true; +} + +bool BPFDAGToDAGISel::fillConstantArray(const DataLayout &DL, + const ConstantArray *CA, + val_vec_type &Vals, int Offset) { + for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i) { + if (fillGenericConstant(DL, CA->getOperand(i), Vals, Offset) == false) + return false; + Offset += DL.getTypeAllocSize(CA->getOperand(i)->getType()); + } + + return true; +} + +bool BPFDAGToDAGISel::fillConstantStruct(const DataLayout &DL, + const ConstantStruct *CS, + val_vec_type &Vals, int Offset) { + const StructLayout *Layout = DL.getStructLayout(CS->getType()); + for (unsigned i = 0, e = CS->getNumOperands(); i != e; ++i) { + const Constant *Field = CS->getOperand(i); + uint64_t SizeSoFar = Layout->getElementOffset(i); + if (fillGenericConstant(DL, Field, Vals, Offset + SizeSoFar) == false) + return false; + } + return true; +} + FunctionPass *llvm::createBPFISelDag(BPFTargetMachine &TM) { return new BPFDAGToDAGISel(TM); } diff --git a/test/CodeGen/BPF/rodata_1.ll b/test/CodeGen/BPF/rodata_1.ll new file mode 100644 index 00000000000..5566f76bb75 --- /dev/null +++ b/test/CodeGen/BPF/rodata_1.ll @@ -0,0 +1,52 @@ +; RUN: llc < %s -march=bpfel -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -march=bpfeb -verify-machineinstrs | FileCheck %s + +; Source code: +; struct test_t1 { +; char a, b, c; +; }; +; struct test_t2 { +; int a, b, c, d, e; +; }; +; +; struct test_t1 g1; +; struct test_t2 g2; +; int test() +; { +; struct test_t1 t1 = {.c = 1}; +; struct test_t2 t2 = {.c = 1}; +; g1 = t1; +; g2 = t2; +; return 0; +; } + +%struct.test_t1 = type { i8, i8, i8 } +%struct.test_t2 = type { i32, i32, i32, i32, i32 } + +@test.t1 = private unnamed_addr constant %struct.test_t1 { i8 0, i8 0, i8 1 }, align 1 +@test.t2 = private unnamed_addr constant %struct.test_t2 { i32 0, i32 0, i32 1, i32 0, i32 0 }, align 4 +@g1 = common local_unnamed_addr global %struct.test_t1 zeroinitializer, align 1 +@g2 = common local_unnamed_addr global %struct.test_t2 zeroinitializer, align 4 + +; Function Attrs: nounwind +define i32 @test() local_unnamed_addr #0 { +; CHECK-LABEL: test: + +entry: + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* getelementptr inbounds (%struct.test_t1, %struct.test_t1* @g1, i64 0, i32 0), i8* getelementptr inbounds (%struct.test_t1, %struct.test_t1* @test.t1, i64 0, i32 0), i64 3, i32 1, i1 false) + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* bitcast (%struct.test_t2* @g2 to i8*), i8* bitcast (%struct.test_t2* @test.t2 to i8*), i64 20, i32 4, i1 false) +; CHECK: r1 = <MCOperand Expr:(g1)>ll +; CHECK: r2 = 0 +; CHECK: *(u8 *)(r1 + 1) = r2 +; CHECK: r3 = 1 +; CHECK: *(u8 *)(r1 + 2) = r3 +; CHECK: r1 = <MCOperand Expr:(g2)>ll +; CHECK: *(u32 *)(r1 + 8) = r3 + ret i32 0 +} +; CHECK: .section .rodata,"a",@progbits + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1 + +attributes #0 = { nounwind } +attributes #1 = { argmemonly nounwind } diff --git a/test/CodeGen/BPF/rodata_2.ll b/test/CodeGen/BPF/rodata_2.ll new file mode 100644 index 00000000000..74b3c3640c3 --- /dev/null +++ b/test/CodeGen/BPF/rodata_2.ll @@ -0,0 +1,51 @@ +; RUN: llc < %s -march=bpfel -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -march=bpfeb -verify-machineinstrs | FileCheck %s + +; Source code: +; struct test_t1 { +; char a; +; int b; +; }; +; struct test_t2 { +; char a, b; +; struct test_t1 c[2]; +; int d[2]; +; int e; +; }; +; struct test_t2 g; +; int test() +; { +; struct test_t2 t2 = {.c = {{}, {.b = 1}}, .d = {2, 3}}; +; g = t2; +; return 0; +; } + +%struct.test_t2 = type { i8, i8, [2 x %struct.test_t1], [2 x i32], i32 } +%struct.test_t1 = type { i8, i32 } + +@test.t2 = private unnamed_addr constant %struct.test_t2 { i8 0, i8 0, [2 x %struct.test_t1] [%struct.test_t1 zeroinitializer, %struct.test_t1 { i8 0, i32 1 }], [2 x i32] [i32 2, i32 3], i32 0 }, align 4 +@g = common local_unnamed_addr global %struct.test_t2 zeroinitializer, align 4 + +; Function Attrs: nounwind +define i32 @test() local_unnamed_addr #0 { +; CHECK-LABEL: test: + +entry: + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* getelementptr inbounds (%struct.test_t2, %struct.test_t2* @g, i64 0, i32 0), i8* getelementptr inbounds (%struct.test_t2, %struct.test_t2* @test.t2, i64 0, i32 0), i64 32, i32 4, i1 false) +; CHECK: r1 = <MCOperand Expr:(g)>ll +; CHECK: r2 = 0 +; CHECK: *(u32 *)(r1 + 28) = r2 +; CHECK: r3 = 3 +; CHECK: *(u32 *)(r1 + 24) = r3 +; CHECK: r3 = 2 +; CHECK: *(u32 *)(r1 + 20) = r3 +; CHECK: r3 = 1 +; CHECK: *(u32 *)(r1 + 16) = r3 + ret i32 0 +} +; CHECK: .section .rodata.cst32,"aM",@progbits,32 + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1 + +attributes #0 = { nounwind } +attributes #1 = { argmemonly nounwind } diff --git a/test/CodeGen/BPF/rodata_3.ll b/test/CodeGen/BPF/rodata_3.ll new file mode 100644 index 00000000000..814ce764546 --- /dev/null +++ b/test/CodeGen/BPF/rodata_3.ll @@ -0,0 +1,41 @@ +; REQUIRES: x86_64-linux +; RUN: llc < %s -march=bpfel -verify-machineinstrs | FileCheck --check-prefix=CHECK-EL %s +; RUN: llc < %s -march=bpfeb -verify-machineinstrs | FileCheck --check-prefix=CHECK-EB %s +; +; This test requires little-endian host, so we specific x86_64-linux here. +; Source code: +; struct test_t1 { +; char a; +; int b, c, d; +; }; +; +; struct test_t1 g; +; int test() +; { +; struct test_t1 t1 = {.a = 1}; +; g = t1; +; return 0; +; } + +%struct.test_t1 = type { i8, i32, i32, i32 } + +@test.t1 = private unnamed_addr constant %struct.test_t1 { i8 1, i32 0, i32 0, i32 0 }, align 4 +@g = common local_unnamed_addr global %struct.test_t1 zeroinitializer, align 4 + +; Function Attrs: nounwind +define i32 @test() local_unnamed_addr #0 { +entry: + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* getelementptr inbounds (%struct.test_t1, %struct.test_t1* @g, i64 0, i32 0), i8* getelementptr inbounds (%struct.test_t1, %struct.test_t1* @test.t1, i64 0, i32 0), i64 16, i32 4, i1 false) +; CHECK-EL: r2 = 1 +; CHECK-EL: *(u32 *)(r1 + 0) = r2 +; CHECK-EB: r2 = 16777216 +; CHECK-EB: *(u32 *)(r1 + 0) = r2 + ret i32 0 +} +; CHECK-EL: .section .rodata.cst16,"aM",@progbits,16 +; CHECK-EB: .section .rodata.cst16,"aM",@progbits,16 + +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1 + +attributes #0 = { nounwind } +attributes #1 = { argmemonly nounwind } diff --git a/test/CodeGen/BPF/rodata_4.ll b/test/CodeGen/BPF/rodata_4.ll new file mode 100644 index 00000000000..d6b9fba5be0 --- /dev/null +++ b/test/CodeGen/BPF/rodata_4.ll @@ -0,0 +1,43 @@ +; RUN: llc < %s -march=bpfel -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -march=bpfeb -verify-machineinstrs | FileCheck %s + +; Source code: +; struct test_t1 +; { +; short a; +; short b; +; char c; +; }; +; +; struct test_t1 g; +; int test() +; { +; struct test_t1 t1[] = {{50, 500, 5}, {60, 600, 6}, {70, 700, 7}, {80, 800, 8} }; +; +; g = t1[1]; +; return 0; +; } + +%struct.test_t1 = type { i16, i16, i8 } + +@test.t1 = private unnamed_addr constant [4 x %struct.test_t1] [%struct.test_t1 { i16 50, i16 500, i8 5 }, %struct.test_t1 { i16 60, i16 600, i8 6 }, %struct.test_t1 { i16 70, i16 700, i8 7 }, %struct.test_t1 { i16 80, i16 800, i8 8 }], align 2 +@g = common local_unnamed_addr global %struct.test_t1 zeroinitializer, align 2 + +; Function Attrs: nounwind +define i32 @test() local_unnamed_addr #0 { +; CHECK-LABEL: test: +entry: + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* bitcast (%struct.test_t1* @g to i8*), i8* bitcast (%struct.test_t1* getelementptr inbounds ([4 x %struct.test_t1], [4 x %struct.test_t1]* @test.t1, i64 0, i64 1) to i8*), i64 6, i32 2, i1 false) +; CHECK: r2 = 600 +; CHECK: *(u16 *)(r1 + 2) = r2 +; CHECK: r2 = 60 +; CHECK: *(u16 *)(r1 + 0) = r2 + ret i32 0 +} +; CHECK .section .rodata,"a",@progbits + +; Function Attrs: argmemonly nounwind +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1 + +attributes #0 = { nounwind } +attributes #1 = { argmemonly nounwind } |