Merge tag 'dm-4.4-fixes-2' of git://git.kernel.org/pub/scm/linux/kernel/git/device...

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / net / bpf_jit_comp.c

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index cf3c7d4a1b585a928d9b4451ee268141d95d7088..b162ad70effcfeacbc7a304d569b222717f495d7 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -50,7 +50,7 @@ static const int bpf2a64[] = {
        [BPF_REG_8] = A64_R(21),
        [BPF_REG_9] = A64_R(22),
        /* read-only frame pointer to access stack */
-       [BPF_REG_FP] = A64_FP,
+       [BPF_REG_FP] = A64_R(25),
        /* temporary register for internal BPF JIT */
        [TMP_REG_1] = A64_R(23),
        [TMP_REG_2] = A64_R(24),
@@ -139,6 +139,12 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
 /* Stack must be multiples of 16B */
 #define STACK_ALIGN(sz) (((sz) + 15) & ~15)
 
+#define _STACK_SIZE \
+       (MAX_BPF_STACK \
+        + 4 /* extra for skb_copy_bits buffer */)
+
+#define STACK_SIZE STACK_ALIGN(_STACK_SIZE)
+
 static void build_prologue(struct jit_ctx *ctx)
 {
        const u8 r6 = bpf2a64[BPF_REG_6];
@@ -150,10 +156,35 @@ static void build_prologue(struct jit_ctx *ctx)
        const u8 rx = bpf2a64[BPF_REG_X];
        const u8 tmp1 = bpf2a64[TMP_REG_1];
        const u8 tmp2 = bpf2a64[TMP_REG_2];
-       int stack_size = MAX_BPF_STACK;
 
-       stack_size += 4; /* extra for skb_copy_bits buffer */
-       stack_size = STACK_ALIGN(stack_size);
+       /*
+        * BPF prog stack layout
+        *
+        *                         high
+        * original A64_SP =>   0:+-----+ BPF prologue
+        *                        |FP/LR|
+        * current A64_FP =>  -16:+-----+
+        *                        | ... | callee saved registers
+        *                        +-----+
+        *                        |     | x25/x26
+        * BPF fp register => -80:+-----+ <= (BPF_FP)
+        *                        |     |
+        *                        | ... | BPF prog stack
+        *                        |     |
+        *                        +-----+ <= (BPF_FP - MAX_BPF_STACK)
+        *                        |RSVD | JIT scratchpad
+        * current A64_SP =>      +-----+ <= (BPF_FP - STACK_SIZE)
+        *                        |     |
+        *                        | ... | Function call stack
+        *                        |     |
+        *                        +-----+
+        *                          low
+        *
+        */
+
+       /* Save FP and LR registers to stay align with ARM64 AAPCS */
+       emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
+       emit(A64_MOV(1, A64_FP, A64_SP), ctx);
 
        /* Save callee-saved register */
        emit(A64_PUSH(r6, r7, A64_SP), ctx);
@@ -161,12 +192,15 @@ static void build_prologue(struct jit_ctx *ctx)
        if (ctx->tmp_used)
                emit(A64_PUSH(tmp1, tmp2, A64_SP), ctx);
 
-       /* Set up BPF stack */
-       emit(A64_SUB_I(1, A64_SP, A64_SP, stack_size), ctx);
+       /* Save fp (x25) and x26. SP requires 16 bytes alignment */
+       emit(A64_PUSH(fp, A64_R(26), A64_SP), ctx);
 
-       /* Set up frame pointer */
+       /* Set up BPF prog stack base register (x25) */
        emit(A64_MOV(1, fp, A64_SP), ctx);
 
+       /* Set up function call stack */
+       emit(A64_SUB_I(1, A64_SP, A64_SP, STACK_SIZE), ctx);
+
        /* Clear registers A and X */
        emit_a64_mov_i64(ra, 0, ctx);
        emit_a64_mov_i64(rx, 0, ctx);
@@ -182,13 +216,12 @@ static void build_epilogue(struct jit_ctx *ctx)
        const u8 fp = bpf2a64[BPF_REG_FP];
        const u8 tmp1 = bpf2a64[TMP_REG_1];
        const u8 tmp2 = bpf2a64[TMP_REG_2];
-       int stack_size = MAX_BPF_STACK;
-
-       stack_size += 4; /* extra for skb_copy_bits buffer */
-       stack_size = STACK_ALIGN(stack_size);
 
        /* We're done with BPF stack */
-       emit(A64_ADD_I(1, A64_SP, A64_SP, stack_size), ctx);
+       emit(A64_ADD_I(1, A64_SP, A64_SP, STACK_SIZE), ctx);
+
+       /* Restore fs (x25) and x26 */
+       emit(A64_POP(fp, A64_R(26), A64_SP), ctx);
 
        /* Restore callee-saved register */
        if (ctx->tmp_used)
@@ -196,8 +229,8 @@ static void build_epilogue(struct jit_ctx *ctx)
        emit(A64_POP(r8, r9, A64_SP), ctx);
        emit(A64_POP(r6, r7, A64_SP), ctx);
 
-       /* Restore frame pointer */
-       emit(A64_MOV(1, fp, A64_SP), ctx);
+       /* Restore FP/LR registers */
+       emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
 
        /* Set return value */
        emit(A64_MOV(1, A64_R(0), r0), ctx);
@@ -557,7 +590,25 @@ emit_cond_jmp:
        case BPF_ST | BPF_MEM | BPF_H:
        case BPF_ST | BPF_MEM | BPF_B:
        case BPF_ST | BPF_MEM | BPF_DW:
-               goto notyet;
+               /* Load imm to a register then store it */
+               ctx->tmp_used = 1;
+               emit_a64_mov_i(1, tmp2, off, ctx);
+               emit_a64_mov_i(1, tmp, imm, ctx);
+               switch (BPF_SIZE(code)) {
+               case BPF_W:
+                       emit(A64_STR32(tmp, dst, tmp2), ctx);
+                       break;
+               case BPF_H:
+                       emit(A64_STRH(tmp, dst, tmp2), ctx);
+                       break;
+               case BPF_B:
+                       emit(A64_STRB(tmp, dst, tmp2), ctx);
+                       break;
+               case BPF_DW:
+                       emit(A64_STR64(tmp, dst, tmp2), ctx);
+                       break;
+               }
+               break;
 
        /* STX: *(size *)(dst + off) = src */
        case BPF_STX | BPF_MEM | BPF_W:
@@ -624,7 +675,7 @@ emit_cond_jmp:
                        return -EINVAL;
                }
                emit_a64_mov_i64(r3, size, ctx);
-               emit(A64_ADD_I(1, r4, fp, MAX_BPF_STACK), ctx);
+               emit(A64_SUB_I(1, r4, fp, STACK_SIZE), ctx);
                emit_a64_mov_i64(r5, (unsigned long)bpf_load_pointer, ctx);
                emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
                emit(A64_MOV(1, A64_FP, A64_SP), ctx);
@@ -758,7 +809,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog)
        if (bpf_jit_enable > 1)
                bpf_jit_dump(prog->len, image_size, 2, ctx.image);
 
-       bpf_flush_icache(ctx.image, ctx.image + ctx.idx);
+       bpf_flush_icache(header, ctx.image + ctx.idx);
 
        set_memory_ro((unsigned long)header, header->pages);
        prog->bpf_func = (void *)ctx.image;