src/bsp/lk/arch/arm64/cache-ops.S

/*
 * Copyright (c) 2014, Google Inc. All rights reserved
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files
 * (the "Software"), to deal in the Software without restriction,
 * including without limitation the rights to use, copy, modify, merge,
 * publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include <asm.h>
#include <arch/ops.h>
#include <arch/defines.h>

#define LOC_SHIFT           24
#define CLIDR_FIELD_WIDTH   3
#define LEVEL_SHIFT         1
#define DCISW               0x0
#define DCCISW              0x1

.text

.macro cache_range_op, cache op
    add     x2, x0, x1                  // calculate the end address
    bic     x3, x0, #(CACHE_LINE-1)     // align the start with a cache line
.Lcache_range_op_loop\@:
    \cache  \op, x3
    add     x3, x3, #CACHE_LINE
    cmp     x3, x2
    blo     .Lcache_range_op_loop\@
    dsb     sy
.endm

    /* void arch_flush_cache_range(addr_t start, size_t len); */
FUNCTION(arch_clean_cache_range)
    cache_range_op dc cvac         // clean cache to PoC by MVA
    ret

    /* void arch_flush_invalidate_cache_range(addr_t start, size_t len); */
FUNCTION(arch_clean_invalidate_cache_range)
    cache_range_op dc civac        // clean & invalidate dcache to PoC by MVA
    ret

    /* void arch_invalidate_cache_range(addr_t start, size_t len); */
FUNCTION(arch_invalidate_cache_range)
    cache_range_op dc ivac         // invalidate dcache to PoC by MVA
    ret

    /* void arch_sync_cache_range(addr_t start, size_t len); */
FUNCTION(arch_sync_cache_range)
    cache_range_op dc cvau         // clean dcache to PoU by MVA
    cache_range_op ic ivau         // invalidate icache to PoU by MVA
    ret

/* will trash x0-x2, x4-x9, x11, x14, x16-x17 */
LOCAL_FUNCTION(do_dcsw_op)
        cbz     x3, exit
        adr     x14, dcsw_loop_table    // compute inner loop address
        add     x14, x14, x0, lsl #5    // inner loop is 8x32-bit instructions
        mov     x0, x9
        mov     w8, #1
loop1:
        add     x2, x10, x10, lsr #1    // work out 3x current cache level
        lsr     x1, x0, x2              // extract cache type bits from clidr
        and     x1, x1, #7              // mask the bits for current cache only
        cmp     x1, #2                  // see what cache we have at this level
        b.lt    level_done              // nothing to do if no cache or icache

        msr     csselr_el1, x10         // select current cache level in csselr
        isb                             // isb to sych the new cssr&csidr
        mrs     x1, ccsidr_el1          // read the new ccsidr
        and     x2, x1, #7              // extract the length of the cache lines
        add     x2, x2, #4              // add 4 (line length offset)
        ubfx    x4, x1, #3, #10         // maximum way number
        clz     w5, w4                  // bit position of way size increment
        lsl     w9, w4, w5              // w9 = aligned max way number
        lsl     w16, w8, w5             // w16 = way number loop decrement
        orr     w9, w10, w9             // w9 = combine way and cache number
        ubfx    w6, w1, #13, #15        // w6 = max set number
        lsl     w17, w8, w2             // w17 = set number loop decrement
        dsb     sy                      // barrier before we start this level
        br      x14                     // jump to DC operation specific loop

        .macro  dcsw_loop _op
loop2_\_op:
        lsl     w7, w6, w2              // w7 = aligned max set number

loop3_\_op:
        orr     w11, w9, w7             // combine cache, way and set number
        dc      \_op, x11
        subs    w7, w7, w17             // decrement set number
        b.ge    loop3_\_op

        subs    x9, x9, x16             // decrement way number
        b.ge    loop2_\_op

        b       level_done
.endm

level_done:
        add     x10, x10, #2            // increment cache number
        cmp     x3, x10
        b.gt    loop1
        msr     csselr_el1, xzr         // select cache level 0 in csselr
        dsb     sy                      // barrier to complete final cache operation
        isb
exit:
        ret

dcsw_loop_table:
        dcsw_loop isw
        dcsw_loop cisw
        dcsw_loop csw

/* will trash x3, x9, x10 */
.macro  dcsw_op shift, fw, ls
        mrs     x9, clidr_el1
        ubfx    x3, x9, \shift, \fw
        lsl     x3, x3, \ls
        mov     x10, xzr
        bl      do_dcsw_op
.endm

/* void arch_enable_cache(uint flags);
 * For EL1  only.
 */
FUNCTION(arch_enable_cache)
    stp     x29, x30, [sp, #-32]!
    stp     x24, x25, [sp, #16]

    mov     x25, x0
    /* check DCACHE flag */
    tst     x25, #DCACHE
    b.eq    .L__enable_icache
    mrs     x24, sctlr_el1
    tst     x24, #(1<<2)
    b.ne    .L__enable_icache

    /* invalidate dcache */
    mov     x0, #DCISW
    dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT

    /* enable dcache enable bit */
    orr     x24, x24, #(1<<2)
    msr     sctlr_el1, x24

.L__enable_icache:
    /* check ICACHE flag */
    tst     x25, #ICACHE
    b.eq    .L__done_enable
    mrs     x24, sctlr_el1
    tst     x24, #(1<<12)
    b.ne    .L__done_enable

    /* invalidate icache */
    dsb     sy
    ic      iallu
    dsb     sy
    isb

    /* enable icache enable bit */
    mrs     x24, sctlr_el1
    orr     x24, x24, #(1<<12)
    msr     sctlr_el1, x24

.L__done_enable:
    ldp     x24, x25, [sp, #16]
    ldp     x29, x30, [sp], #32
    ret

/* void arch_disable_cache(uint flags) */
/* only for el1 here */
FUNCTION(arch_disable_cache)
    stp     x29, x30, [sp, #-32]!
    str     x25, [sp, #16]

    mov     x25, x0
    /* check DCACHE flag */
    tst     x25, #DCACHE
    b.eq    .L__disable_icache
    mrs     x1, sctlr_el1
    tst     x1, #(1<<2)
    b.eq    .L__dcache_already_disabled

    /* disable dcache enable bit */
    bic     x1, x1, #(1<<2)
    msr     sctlr_el1, x1

    /* clean & invalidate dcache */
    mov     x0, #DCCISW
    b       .L__flush_dcache

.L__dcache_already_disabled:
    /* invalidate dcache */
    mov     x0, #DCISW
.L__flush_dcache:
    dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT

.L__disable_icache:
    /* check ICACHE flag */
    tst     x25, #ICACHE
    b.eq    .L__done_disable
    /* disable icache enable bit */
    mrs     x1, sctlr_el1
    bic     x1, x1, #(1<<12)
    msr     sctlr_el1, x1

    /* invalidate icache for PE to PoU */
    dsb     sy
    ic      iallu
    dsb     sy
    isb

.L__done_disable:
    ldr     x25, [sp, #16]
    ldp     x29, x30, [sp], #32
    ret