| // SPDX-License-Identifier: GPL-2.0-or-later | |
| /* | |
| * AMD Address Translation Library | |
| * | |
| * denormalize.c : Functions to account for interleaving bits | |
| * | |
| * Copyright (c) 2023, Advanced Micro Devices, Inc. | |
| * All Rights Reserved. | |
| * | |
| * Author: Yazen Ghannam <Yazen.Ghannam@amd.com> | |
| */ | |
| #include "internal.h" | |
| /* | |
| * Returns the Destination Fabric ID. This is the first (lowest) | |
| * COH_ST Fabric ID used within a DRAM Address map. | |
| */ | |
| static u16 get_dst_fabric_id(struct addr_ctx *ctx) | |
| { | |
| switch (df_cfg.rev) { | |
| case DF2: return FIELD_GET(DF2_DST_FABRIC_ID, ctx->map.limit); | |
| case DF3: return FIELD_GET(DF3_DST_FABRIC_ID, ctx->map.limit); | |
| case DF3p5: return FIELD_GET(DF3p5_DST_FABRIC_ID, ctx->map.limit); | |
| case DF4: return FIELD_GET(DF4_DST_FABRIC_ID, ctx->map.ctl); | |
| case DF4p5: return FIELD_GET(DF4p5_DST_FABRIC_ID, ctx->map.ctl); | |
| default: | |
| atl_debug_on_bad_df_rev(); | |
| return 0; | |
| } | |
| } | |
| /* | |
| * Make a contiguous gap in address for N bits starting at bit P. | |
| * | |
| * Example: | |
| * address bits: [20:0] | |
| * # of interleave bits (n): 3 | |
| * starting interleave bit (p): 8 | |
| * | |
| * expanded address bits: [20+n : n+p][n+p-1 : p][p-1 : 0] | |
| * [23 : 11][10 : 8][7 : 0] | |
| */ | |
| static u64 make_space_for_coh_st_id_at_intlv_bit(struct addr_ctx *ctx) | |
| { | |
| return expand_bits(ctx->map.intlv_bit_pos, | |
| ctx->map.total_intlv_bits, | |
| ctx->ret_addr); | |
| } | |
| /* | |
| * Make two gaps in address for N bits. | |
| * First gap is a single bit at bit P. | |
| * Second gap is the remaining N-1 bits at bit 12. | |
| * | |
| * Example: | |
| * address bits: [20:0] | |
| * # of interleave bits (n): 3 | |
| * starting interleave bit (p): 8 | |
| * | |
| * First gap | |
| * expanded address bits: [20+1 : p+1][p][p-1 : 0] | |
| * [21 : 9][8][7 : 0] | |
| * | |
| * Second gap uses result from first. | |
| * r = n - 1; remaining interleave bits | |
| * expanded address bits: [21+r : 12+r][12+r-1: 12][11 : 0] | |
| * [23 : 14][13 : 12][11 : 0] | |
| */ | |
| static u64 make_space_for_coh_st_id_split_2_1(struct addr_ctx *ctx) | |
| { | |
| /* Make a single space at the interleave bit. */ | |
| u64 denorm_addr = expand_bits(ctx->map.intlv_bit_pos, 1, ctx->ret_addr); | |
| /* Done if there's only a single interleave bit. */ | |
| if (ctx->map.total_intlv_bits <= 1) | |
| return denorm_addr; | |
| /* Make spaces for the remaining interleave bits starting at bit 12. */ | |
| return expand_bits(12, ctx->map.total_intlv_bits - 1, denorm_addr); | |
| } | |
| /* | |
| * Make space for CS ID at bits [14:8] as follows: | |
| * | |
| * 8 channels -> bits [10:8] | |
| * 16 channels -> bits [11:8] | |
| * 32 channels -> bits [14,11:8] | |
| * | |
| * 1 die -> N/A | |
| * 2 dies -> bit [12] | |
| * 4 dies -> bits [13:12] | |
| */ | |
| static u64 make_space_for_coh_st_id_mi300(struct addr_ctx *ctx) | |
| { | |
| u8 num_intlv_bits = ilog2(ctx->map.num_intlv_chan); | |
| u64 denorm_addr; | |
| if (ctx->map.intlv_bit_pos != 8) { | |
| pr_debug("Invalid interleave bit: %u", ctx->map.intlv_bit_pos); | |
| return ~0ULL; | |
| } | |
| /* Channel bits. Covers up to 4 bits at [11:8]. */ | |
| denorm_addr = expand_bits(8, min(num_intlv_bits, 4), ctx->ret_addr); | |
| /* Die bits. Always starts at [12]. */ | |
| denorm_addr = expand_bits(12, ilog2(ctx->map.num_intlv_dies), denorm_addr); | |
| /* Additional channel bit at [14]. */ | |
| if (num_intlv_bits > 4) | |
| denorm_addr = expand_bits(14, 1, denorm_addr); | |
| return denorm_addr; | |
| } | |
| /* | |
| * Take the current calculated address and shift enough bits in the middle | |
| * to make a gap where the interleave bits will be inserted. | |
| */ | |
| static u64 make_space_for_coh_st_id(struct addr_ctx *ctx) | |
| { | |
| switch (ctx->map.intlv_mode) { | |
| case NOHASH_2CHAN: | |
| case NOHASH_4CHAN: | |
| case NOHASH_8CHAN: | |
| case NOHASH_16CHAN: | |
| case NOHASH_32CHAN: | |
| case DF2_2CHAN_HASH: | |
| return make_space_for_coh_st_id_at_intlv_bit(ctx); | |
| case DF3_COD4_2CHAN_HASH: | |
| case DF3_COD2_4CHAN_HASH: | |
| case DF3_COD1_8CHAN_HASH: | |
| case DF4_NPS4_2CHAN_HASH: | |
| case DF4_NPS2_4CHAN_HASH: | |
| case DF4_NPS1_8CHAN_HASH: | |
| case DF4p5_NPS4_2CHAN_1K_HASH: | |
| case DF4p5_NPS4_2CHAN_2K_HASH: | |
| case DF4p5_NPS2_4CHAN_2K_HASH: | |
| case DF4p5_NPS1_8CHAN_2K_HASH: | |
| case DF4p5_NPS1_16CHAN_2K_HASH: | |
| return make_space_for_coh_st_id_split_2_1(ctx); | |
| case MI3_HASH_8CHAN: | |
| case MI3_HASH_16CHAN: | |
| case MI3_HASH_32CHAN: | |
| return make_space_for_coh_st_id_mi300(ctx); | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return ~0ULL; | |
| } | |
| } | |
| static u16 get_coh_st_id_df2(struct addr_ctx *ctx) | |
| { | |
| u8 num_socket_intlv_bits = ilog2(ctx->map.num_intlv_sockets); | |
| u8 num_die_intlv_bits = ilog2(ctx->map.num_intlv_dies); | |
| u8 num_intlv_bits; | |
| u16 coh_st_id, mask; | |
| coh_st_id = ctx->coh_st_fabric_id - get_dst_fabric_id(ctx); | |
| /* Channel interleave bits */ | |
| num_intlv_bits = order_base_2(ctx->map.num_intlv_chan); | |
| mask = GENMASK(num_intlv_bits - 1, 0); | |
| coh_st_id &= mask; | |
| /* Die interleave bits */ | |
| if (num_die_intlv_bits) { | |
| u16 die_bits; | |
| mask = GENMASK(num_die_intlv_bits - 1, 0); | |
| die_bits = ctx->coh_st_fabric_id & df_cfg.die_id_mask; | |
| die_bits >>= df_cfg.die_id_shift; | |
| coh_st_id |= (die_bits & mask) << num_intlv_bits; | |
| num_intlv_bits += num_die_intlv_bits; | |
| } | |
| /* Socket interleave bits */ | |
| if (num_socket_intlv_bits) { | |
| u16 socket_bits; | |
| mask = GENMASK(num_socket_intlv_bits - 1, 0); | |
| socket_bits = ctx->coh_st_fabric_id & df_cfg.socket_id_mask; | |
| socket_bits >>= df_cfg.socket_id_shift; | |
| coh_st_id |= (socket_bits & mask) << num_intlv_bits; | |
| } | |
| return coh_st_id; | |
| } | |
| static u16 get_coh_st_id_df4(struct addr_ctx *ctx) | |
| { | |
| /* | |
| * Start with the original component mask and the number of interleave | |
| * bits for the channels in this map. | |
| */ | |
| u8 num_intlv_bits = ilog2(ctx->map.num_intlv_chan); | |
| u16 mask = df_cfg.component_id_mask; | |
| u16 socket_bits; | |
| /* Set the derived Coherent Station ID to the input Coherent Station Fabric ID. */ | |
| u16 coh_st_id = ctx->coh_st_fabric_id & mask; | |
| /* | |
| * Subtract the "base" Destination Fabric ID. | |
| * This accounts for systems with disabled Coherent Stations. | |
| */ | |
| coh_st_id -= get_dst_fabric_id(ctx) & mask; | |
| /* | |
| * Generate and use a new mask based on the number of bits | |
| * needed for channel interleaving in this map. | |
| */ | |
| mask = GENMASK(num_intlv_bits - 1, 0); | |
| coh_st_id &= mask; | |
| /* Done if socket interleaving is not enabled. */ | |
| if (ctx->map.num_intlv_sockets <= 1) | |
| return coh_st_id; | |
| /* | |
| * Figure out how many bits are needed for the number of | |
| * interleaved sockets. And shift the derived Coherent Station ID to account | |
| * for these. | |
| */ | |
| num_intlv_bits = ilog2(ctx->map.num_intlv_sockets); | |
| coh_st_id <<= num_intlv_bits; | |
| /* Generate a new mask for the socket interleaving bits. */ | |
| mask = GENMASK(num_intlv_bits - 1, 0); | |
| /* Get the socket interleave bits from the original Coherent Station Fabric ID. */ | |
| socket_bits = (ctx->coh_st_fabric_id & df_cfg.socket_id_mask) >> df_cfg.socket_id_shift; | |
| /* Apply the appropriate socket bits to the derived Coherent Station ID. */ | |
| coh_st_id |= socket_bits & mask; | |
| return coh_st_id; | |
| } | |
| /* | |
| * MI300 hash has: | |
| * (C)hannel[3:0] = coh_st_id[3:0] | |
| * (S)tack[0] = coh_st_id[4] | |
| * (D)ie[1:0] = coh_st_id[6:5] | |
| * | |
| * Hashed coh_st_id is swizzled so that Stack bit is at the end. | |
| * coh_st_id = SDDCCCC | |
| */ | |
| static u16 get_coh_st_id_mi300(struct addr_ctx *ctx) | |
| { | |
| u8 channel_bits, die_bits, stack_bit; | |
| u16 die_id; | |
| /* Subtract the "base" Destination Fabric ID. */ | |
| ctx->coh_st_fabric_id -= get_dst_fabric_id(ctx); | |
| die_id = (ctx->coh_st_fabric_id & df_cfg.die_id_mask) >> df_cfg.die_id_shift; | |
| channel_bits = FIELD_GET(GENMASK(3, 0), ctx->coh_st_fabric_id); | |
| stack_bit = FIELD_GET(BIT(4), ctx->coh_st_fabric_id) << 6; | |
| die_bits = die_id << 4; | |
| return stack_bit | die_bits | channel_bits; | |
| } | |
| /* | |
| * Derive the correct Coherent Station ID that represents the interleave bits | |
| * used within the system physical address. This accounts for the | |
| * interleave mode, number of interleaved channels/dies/sockets, and | |
| * other system/mode-specific bit swizzling. | |
| * | |
| * Returns: Coherent Station ID on success. | |
| * All bits set on error. | |
| */ | |
| static u16 calculate_coh_st_id(struct addr_ctx *ctx) | |
| { | |
| switch (ctx->map.intlv_mode) { | |
| case NOHASH_2CHAN: | |
| case NOHASH_4CHAN: | |
| case NOHASH_8CHAN: | |
| case NOHASH_16CHAN: | |
| case NOHASH_32CHAN: | |
| case DF3_COD4_2CHAN_HASH: | |
| case DF3_COD2_4CHAN_HASH: | |
| case DF3_COD1_8CHAN_HASH: | |
| case DF2_2CHAN_HASH: | |
| return get_coh_st_id_df2(ctx); | |
| case DF4_NPS4_2CHAN_HASH: | |
| case DF4_NPS2_4CHAN_HASH: | |
| case DF4_NPS1_8CHAN_HASH: | |
| case DF4p5_NPS4_2CHAN_1K_HASH: | |
| case DF4p5_NPS4_2CHAN_2K_HASH: | |
| case DF4p5_NPS2_4CHAN_2K_HASH: | |
| case DF4p5_NPS1_8CHAN_2K_HASH: | |
| case DF4p5_NPS1_16CHAN_2K_HASH: | |
| return get_coh_st_id_df4(ctx); | |
| case MI3_HASH_8CHAN: | |
| case MI3_HASH_16CHAN: | |
| case MI3_HASH_32CHAN: | |
| return get_coh_st_id_mi300(ctx); | |
| /* COH_ST ID is simply the COH_ST Fabric ID adjusted by the Destination Fabric ID. */ | |
| case DF4p5_NPS2_4CHAN_1K_HASH: | |
| case DF4p5_NPS1_8CHAN_1K_HASH: | |
| case DF4p5_NPS1_16CHAN_1K_HASH: | |
| return ctx->coh_st_fabric_id - get_dst_fabric_id(ctx); | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return ~0; | |
| } | |
| } | |
| static u64 insert_coh_st_id_at_intlv_bit(struct addr_ctx *ctx, u64 denorm_addr, u16 coh_st_id) | |
| { | |
| return denorm_addr | (coh_st_id << ctx->map.intlv_bit_pos); | |
| } | |
| static u64 insert_coh_st_id_split_2_1(struct addr_ctx *ctx, u64 denorm_addr, u16 coh_st_id) | |
| { | |
| /* Insert coh_st_id[0] at the interleave bit. */ | |
| denorm_addr |= (coh_st_id & BIT(0)) << ctx->map.intlv_bit_pos; | |
| /* Insert coh_st_id[2:1] at bit 12. */ | |
| denorm_addr |= (coh_st_id & GENMASK(2, 1)) << 11; | |
| return denorm_addr; | |
| } | |
| static u64 insert_coh_st_id_split_2_2(struct addr_ctx *ctx, u64 denorm_addr, u16 coh_st_id) | |
| { | |
| /* Insert coh_st_id[1:0] at bit 8. */ | |
| denorm_addr |= (coh_st_id & GENMASK(1, 0)) << 8; | |
| /* | |
| * Insert coh_st_id[n:2] at bit 12. 'n' could be 2 or 3. | |
| * Grab both because bit 3 will be clear if unused. | |
| */ | |
| denorm_addr |= (coh_st_id & GENMASK(3, 2)) << 10; | |
| return denorm_addr; | |
| } | |
| static u64 insert_coh_st_id(struct addr_ctx *ctx, u64 denorm_addr, u16 coh_st_id) | |
| { | |
| switch (ctx->map.intlv_mode) { | |
| case NOHASH_2CHAN: | |
| case NOHASH_4CHAN: | |
| case NOHASH_8CHAN: | |
| case NOHASH_16CHAN: | |
| case NOHASH_32CHAN: | |
| case MI3_HASH_8CHAN: | |
| case MI3_HASH_16CHAN: | |
| case MI3_HASH_32CHAN: | |
| case DF2_2CHAN_HASH: | |
| return insert_coh_st_id_at_intlv_bit(ctx, denorm_addr, coh_st_id); | |
| case DF3_COD4_2CHAN_HASH: | |
| case DF3_COD2_4CHAN_HASH: | |
| case DF3_COD1_8CHAN_HASH: | |
| case DF4_NPS4_2CHAN_HASH: | |
| case DF4_NPS2_4CHAN_HASH: | |
| case DF4_NPS1_8CHAN_HASH: | |
| case DF4p5_NPS4_2CHAN_1K_HASH: | |
| case DF4p5_NPS4_2CHAN_2K_HASH: | |
| case DF4p5_NPS2_4CHAN_2K_HASH: | |
| case DF4p5_NPS1_8CHAN_2K_HASH: | |
| case DF4p5_NPS1_16CHAN_2K_HASH: | |
| return insert_coh_st_id_split_2_1(ctx, denorm_addr, coh_st_id); | |
| case DF4p5_NPS2_4CHAN_1K_HASH: | |
| case DF4p5_NPS1_8CHAN_1K_HASH: | |
| case DF4p5_NPS1_16CHAN_1K_HASH: | |
| return insert_coh_st_id_split_2_2(ctx, denorm_addr, coh_st_id); | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return ~0ULL; | |
| } | |
| } | |
| /* | |
| * MI300 systems have a fixed, hardware-defined physical-to-logical | |
| * Coherent Station mapping. The Remap registers are not used. | |
| */ | |
| static const u16 phy_to_log_coh_st_map_mi300[] = { | |
| 12, 13, 14, 15, | |
| 8, 9, 10, 11, | |
| 4, 5, 6, 7, | |
| 0, 1, 2, 3, | |
| 28, 29, 30, 31, | |
| 24, 25, 26, 27, | |
| 20, 21, 22, 23, | |
| 16, 17, 18, 19, | |
| }; | |
| static u16 get_logical_coh_st_fabric_id_mi300(struct addr_ctx *ctx) | |
| { | |
| if (ctx->inst_id >= ARRAY_SIZE(phy_to_log_coh_st_map_mi300)) { | |
| atl_debug(ctx, "Instance ID out of range"); | |
| return ~0; | |
| } | |
| return phy_to_log_coh_st_map_mi300[ctx->inst_id] | (ctx->node_id << df_cfg.node_id_shift); | |
| } | |
| static u16 get_logical_coh_st_fabric_id(struct addr_ctx *ctx) | |
| { | |
| u16 component_id, log_fabric_id; | |
| /* Start with the physical COH_ST Fabric ID. */ | |
| u16 phys_fabric_id = ctx->coh_st_fabric_id; | |
| if (df_cfg.rev == DF4p5 && df_cfg.flags.heterogeneous) | |
| return get_logical_coh_st_fabric_id_mi300(ctx); | |
| /* Skip logical ID lookup if remapping is disabled. */ | |
| if (!FIELD_GET(DF4_REMAP_EN, ctx->map.ctl) && | |
| ctx->map.intlv_mode != DF3_6CHAN) | |
| return phys_fabric_id; | |
| /* Mask off the Node ID bits to get the "local" Component ID. */ | |
| component_id = phys_fabric_id & df_cfg.component_id_mask; | |
| /* | |
| * Search the list of logical Component IDs for the one that | |
| * matches this physical Component ID. | |
| */ | |
| for (log_fabric_id = 0; log_fabric_id < MAX_COH_ST_CHANNELS; log_fabric_id++) { | |
| if (ctx->map.remap_array[log_fabric_id] == component_id) | |
| break; | |
| } | |
| if (log_fabric_id == MAX_COH_ST_CHANNELS) | |
| atl_debug(ctx, "COH_ST remap entry not found for 0x%x", | |
| log_fabric_id); | |
| /* Get the Node ID bits from the physical and apply to the logical. */ | |
| return (phys_fabric_id & df_cfg.node_id_mask) | log_fabric_id; | |
| } | |
| static u16 get_logical_coh_st_fabric_id_for_current_spa(struct addr_ctx *ctx, | |
| struct df4p5_denorm_ctx *denorm_ctx) | |
| { | |
| bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G, hash_ctl_1T; | |
| bool hash_pa8, hash_pa9, hash_pa12, hash_pa13; | |
| u64 cs_id = 0; | |
| hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); | |
| hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); | |
| hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); | |
| hash_ctl_1T = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl); | |
| hash_pa8 = FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa); | |
| hash_pa8 ^= FIELD_GET(BIT_ULL(14), denorm_ctx->current_spa); | |
| hash_pa8 ^= FIELD_GET(BIT_ULL(16), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hash_pa8 ^= FIELD_GET(BIT_ULL(21), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hash_pa8 ^= FIELD_GET(BIT_ULL(30), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hash_pa8 ^= FIELD_GET(BIT_ULL(40), denorm_ctx->current_spa) & hash_ctl_1T; | |
| hash_pa9 = FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa); | |
| hash_pa9 ^= FIELD_GET(BIT_ULL(17), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hash_pa9 ^= FIELD_GET(BIT_ULL(22), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hash_pa9 ^= FIELD_GET(BIT_ULL(31), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hash_pa9 ^= FIELD_GET(BIT_ULL(41), denorm_ctx->current_spa) & hash_ctl_1T; | |
| hash_pa12 = FIELD_GET(BIT_ULL(12), denorm_ctx->current_spa); | |
| hash_pa12 ^= FIELD_GET(BIT_ULL(18), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hash_pa12 ^= FIELD_GET(BIT_ULL(23), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hash_pa12 ^= FIELD_GET(BIT_ULL(32), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hash_pa12 ^= FIELD_GET(BIT_ULL(42), denorm_ctx->current_spa) & hash_ctl_1T; | |
| hash_pa13 = FIELD_GET(BIT_ULL(13), denorm_ctx->current_spa); | |
| hash_pa13 ^= FIELD_GET(BIT_ULL(19), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hash_pa13 ^= FIELD_GET(BIT_ULL(24), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hash_pa13 ^= FIELD_GET(BIT_ULL(33), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hash_pa13 ^= FIELD_GET(BIT_ULL(43), denorm_ctx->current_spa) & hash_ctl_1T; | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 13), denorm_ctx->current_spa) << 3; | |
| cs_id %= denorm_ctx->mod_value; | |
| cs_id <<= 2; | |
| cs_id |= (hash_pa9 | (hash_pa12 << 1)); | |
| cs_id |= hash_pa8 << df_cfg.socket_id_shift; | |
| break; | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 14), denorm_ctx->current_spa) << 4; | |
| cs_id %= denorm_ctx->mod_value; | |
| cs_id <<= 2; | |
| cs_id |= (hash_pa12 | (hash_pa13 << 1)); | |
| cs_id |= hash_pa8 << df_cfg.socket_id_shift; | |
| break; | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; | |
| cs_id %= denorm_ctx->mod_value; | |
| cs_id <<= 2; | |
| cs_id |= (hash_pa8 | (hash_pa9 << 1)); | |
| break; | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 13), denorm_ctx->current_spa) << 3; | |
| cs_id %= denorm_ctx->mod_value; | |
| cs_id <<= 2; | |
| cs_id |= (hash_pa8 | (hash_pa12 << 1)); | |
| break; | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; | |
| cs_id |= (FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa) << 1); | |
| cs_id %= denorm_ctx->mod_value; | |
| cs_id <<= 1; | |
| cs_id |= hash_pa8; | |
| break; | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; | |
| cs_id %= denorm_ctx->mod_value; | |
| cs_id <<= 1; | |
| cs_id |= hash_pa8; | |
| break; | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; | |
| cs_id |= FIELD_GET(GENMASK_ULL(9, 8), denorm_ctx->current_spa); | |
| cs_id %= denorm_ctx->mod_value; | |
| break; | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| cs_id = FIELD_GET(GENMASK_ULL(63, 12), denorm_ctx->current_spa) << 2; | |
| cs_id |= FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa) << 1; | |
| cs_id %= denorm_ctx->mod_value; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return 0; | |
| } | |
| if (cs_id > 0xffff) { | |
| atl_debug(ctx, "Translation error: Resulting cs_id larger than u16\n"); | |
| return 0; | |
| } | |
| return cs_id; | |
| } | |
| static int denorm_addr_common(struct addr_ctx *ctx) | |
| { | |
| u64 denorm_addr; | |
| u16 coh_st_id; | |
| /* | |
| * Convert the original physical COH_ST Fabric ID to a logical value. | |
| * This is required for non-power-of-two and other interleaving modes. | |
| */ | |
| ctx->coh_st_fabric_id = get_logical_coh_st_fabric_id(ctx); | |
| denorm_addr = make_space_for_coh_st_id(ctx); | |
| coh_st_id = calculate_coh_st_id(ctx); | |
| ctx->ret_addr = insert_coh_st_id(ctx, denorm_addr, coh_st_id); | |
| return 0; | |
| } | |
| static int denorm_addr_df3_6chan(struct addr_ctx *ctx) | |
| { | |
| u16 coh_st_id = ctx->coh_st_fabric_id & df_cfg.component_id_mask; | |
| u8 total_intlv_bits = ctx->map.total_intlv_bits; | |
| u8 low_bit, intlv_bit = ctx->map.intlv_bit_pos; | |
| u64 msb_intlv_bits, temp_addr_a, temp_addr_b; | |
| u8 np2_bits = ctx->map.np2_bits; | |
| if (ctx->map.intlv_mode != DF3_6CHAN) | |
| return -EINVAL; | |
| /* | |
| * 'np2_bits' holds the number of bits needed to cover the | |
| * amount of memory (rounded up) in this map using 64K chunks. | |
| * | |
| * Example: | |
| * Total memory in map: 6GB | |
| * Rounded up to next power-of-2: 8GB | |
| * Number of 64K chunks: 0x20000 | |
| * np2_bits = log2(# of chunks): 17 | |
| * | |
| * Get the two most-significant interleave bits from the | |
| * input address based on the following: | |
| * | |
| * [15 + np2_bits - total_intlv_bits : 14 + np2_bits - total_intlv_bits] | |
| */ | |
| low_bit = 14 + np2_bits - total_intlv_bits; | |
| msb_intlv_bits = ctx->ret_addr >> low_bit; | |
| msb_intlv_bits &= 0x3; | |
| /* | |
| * If MSB are 11b, then logical COH_ST ID is 6 or 7. | |
| * Need to adjust based on the mod3 result. | |
| */ | |
| if (msb_intlv_bits == 3) { | |
| u8 addr_mod, phys_addr_msb, msb_coh_st_id; | |
| /* Get the remaining interleave bits from the input address. */ | |
| temp_addr_b = GENMASK_ULL(low_bit - 1, intlv_bit) & ctx->ret_addr; | |
| temp_addr_b >>= intlv_bit; | |
| /* Calculate the logical COH_ST offset based on mod3. */ | |
| addr_mod = temp_addr_b % 3; | |
| /* Get COH_ST ID bits [2:1]. */ | |
| msb_coh_st_id = (coh_st_id >> 1) & 0x3; | |
| /* Get the bit that starts the physical address bits. */ | |
| phys_addr_msb = (intlv_bit + np2_bits + 1); | |
| phys_addr_msb &= BIT(0); | |
| phys_addr_msb++; | |
| phys_addr_msb *= 3 - addr_mod + msb_coh_st_id; | |
| phys_addr_msb %= 3; | |
| /* Move the physical address MSB to the correct place. */ | |
| temp_addr_b |= phys_addr_msb << (low_bit - total_intlv_bits - intlv_bit); | |
| /* Generate a new COH_ST ID as follows: coh_st_id = [1, 1, coh_st_id[0]] */ | |
| coh_st_id &= BIT(0); | |
| coh_st_id |= GENMASK(2, 1); | |
| } else { | |
| temp_addr_b = GENMASK_ULL(63, intlv_bit) & ctx->ret_addr; | |
| temp_addr_b >>= intlv_bit; | |
| } | |
| temp_addr_a = GENMASK_ULL(intlv_bit - 1, 0) & ctx->ret_addr; | |
| temp_addr_b <<= intlv_bit + total_intlv_bits; | |
| ctx->ret_addr = temp_addr_a | temp_addr_b; | |
| ctx->ret_addr |= coh_st_id << intlv_bit; | |
| return 0; | |
| } | |
| static int denorm_addr_df4_np2(struct addr_ctx *ctx) | |
| { | |
| bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G; | |
| u16 group, group_offset, log_coh_st_offset; | |
| unsigned int mod_value, shift_value; | |
| u16 mask = df_cfg.component_id_mask; | |
| u64 temp_addr_a, temp_addr_b; | |
| bool hash_pa8, hashed_bit; | |
| switch (ctx->map.intlv_mode) { | |
| case DF4_NPS4_3CHAN_HASH: | |
| mod_value = 3; | |
| shift_value = 13; | |
| break; | |
| case DF4_NPS2_6CHAN_HASH: | |
| mod_value = 3; | |
| shift_value = 12; | |
| break; | |
| case DF4_NPS1_12CHAN_HASH: | |
| mod_value = 3; | |
| shift_value = 11; | |
| break; | |
| case DF4_NPS2_5CHAN_HASH: | |
| mod_value = 5; | |
| shift_value = 13; | |
| break; | |
| case DF4_NPS1_10CHAN_HASH: | |
| mod_value = 5; | |
| shift_value = 12; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return -EINVAL; | |
| } | |
| if (ctx->map.num_intlv_sockets == 1) { | |
| hash_pa8 = BIT_ULL(shift_value) & ctx->ret_addr; | |
| temp_addr_a = remove_bits(shift_value, shift_value, ctx->ret_addr); | |
| } else { | |
| hash_pa8 = ctx->coh_st_fabric_id & df_cfg.socket_id_mask; | |
| temp_addr_a = ctx->ret_addr; | |
| } | |
| /* Make a gap for the real bit [8]. */ | |
| temp_addr_a = expand_bits(8, 1, temp_addr_a); | |
| /* Make an additional gap for bits [13:12], as appropriate.*/ | |
| if (ctx->map.intlv_mode == DF4_NPS2_6CHAN_HASH || | |
| ctx->map.intlv_mode == DF4_NPS1_10CHAN_HASH) { | |
| temp_addr_a = expand_bits(13, 1, temp_addr_a); | |
| } else if (ctx->map.intlv_mode == DF4_NPS1_12CHAN_HASH) { | |
| temp_addr_a = expand_bits(12, 2, temp_addr_a); | |
| } | |
| /* Keep bits [13:0]. */ | |
| temp_addr_a &= GENMASK_ULL(13, 0); | |
| /* Get the appropriate high bits. */ | |
| shift_value += 1 - ilog2(ctx->map.num_intlv_sockets); | |
| temp_addr_b = GENMASK_ULL(63, shift_value) & ctx->ret_addr; | |
| temp_addr_b >>= shift_value; | |
| temp_addr_b *= mod_value; | |
| /* | |
| * Coherent Stations are divided into groups. | |
| * | |
| * Multiples of 3 (mod3) are divided into quadrants. | |
| * e.g. NP4_3CHAN -> [0, 1, 2] [6, 7, 8] | |
| * [3, 4, 5] [9, 10, 11] | |
| * | |
| * Multiples of 5 (mod5) are divided into sides. | |
| * e.g. NP2_5CHAN -> [0, 1, 2, 3, 4] [5, 6, 7, 8, 9] | |
| */ | |
| /* | |
| * Calculate the logical offset for the COH_ST within its DRAM Address map. | |
| * e.g. if map includes [5, 6, 7, 8, 9] and target instance is '8', then | |
| * log_coh_st_offset = 8 - 5 = 3 | |
| */ | |
| log_coh_st_offset = (ctx->coh_st_fabric_id & mask) - (get_dst_fabric_id(ctx) & mask); | |
| /* | |
| * Figure out the group number. | |
| * | |
| * Following above example, | |
| * log_coh_st_offset = 3 | |
| * mod_value = 5 | |
| * group = 3 / 5 = 0 | |
| */ | |
| group = log_coh_st_offset / mod_value; | |
| /* | |
| * Figure out the offset within the group. | |
| * | |
| * Following above example, | |
| * log_coh_st_offset = 3 | |
| * mod_value = 5 | |
| * group_offset = 3 % 5 = 3 | |
| */ | |
| group_offset = log_coh_st_offset % mod_value; | |
| /* Adjust group_offset if the hashed bit [8] is set. */ | |
| if (hash_pa8) { | |
| if (!group_offset) | |
| group_offset = mod_value - 1; | |
| else | |
| group_offset--; | |
| } | |
| /* Add in the group offset to the high bits. */ | |
| temp_addr_b += group_offset; | |
| /* Shift the high bits to the proper starting position. */ | |
| temp_addr_b <<= 14; | |
| /* Combine the high and low bits together. */ | |
| ctx->ret_addr = temp_addr_a | temp_addr_b; | |
| /* Account for hashing here instead of in dehash_address(). */ | |
| hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); | |
| hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); | |
| hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); | |
| hashed_bit = !!hash_pa8; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G; | |
| ctx->ret_addr |= hashed_bit << 8; | |
| /* Done for 3 and 5 channel. */ | |
| if (ctx->map.intlv_mode == DF4_NPS4_3CHAN_HASH || | |
| ctx->map.intlv_mode == DF4_NPS2_5CHAN_HASH) | |
| return 0; | |
| /* Select the proper 'group' bit to use for Bit 13. */ | |
| if (ctx->map.intlv_mode == DF4_NPS1_12CHAN_HASH) | |
| hashed_bit = !!(group & BIT(1)); | |
| else | |
| hashed_bit = group & BIT(0); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G; | |
| ctx->ret_addr |= hashed_bit << 13; | |
| /* Done for 6 and 10 channel. */ | |
| if (ctx->map.intlv_mode != DF4_NPS1_12CHAN_HASH) | |
| return 0; | |
| hashed_bit = group & BIT(0); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G; | |
| ctx->ret_addr |= hashed_bit << 12; | |
| return 0; | |
| } | |
| static u64 normalize_addr_df4p5_np2(struct addr_ctx *ctx, struct df4p5_denorm_ctx *denorm_ctx, | |
| u64 addr) | |
| { | |
| u64 temp_addr_a = 0, temp_addr_b = 0; | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| temp_addr_a = FIELD_GET(GENMASK_ULL(11, 10), addr) << 8; | |
| break; | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| temp_addr_a = FIELD_GET(GENMASK_ULL(11, 9), addr) << 8; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return 0; | |
| } | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 13), addr) / denorm_ctx->mod_value; | |
| temp_addr_b <<= 10; | |
| break; | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 14), addr) / denorm_ctx->mod_value; | |
| temp_addr_b <<= 11; | |
| break; | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) / denorm_ctx->mod_value; | |
| temp_addr_b <<= 10; | |
| break; | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 13), addr) / denorm_ctx->mod_value; | |
| temp_addr_b <<= 11; | |
| break; | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) << 1; | |
| temp_addr_b |= FIELD_GET(BIT_ULL(9), addr); | |
| temp_addr_b /= denorm_ctx->mod_value; | |
| temp_addr_b <<= 10; | |
| break; | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) / denorm_ctx->mod_value; | |
| temp_addr_b <<= 11; | |
| break; | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) << 2; | |
| temp_addr_b |= FIELD_GET(GENMASK_ULL(9, 8), addr); | |
| temp_addr_b /= denorm_ctx->mod_value; | |
| temp_addr_b <<= 10; | |
| break; | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| temp_addr_b = FIELD_GET(GENMASK_ULL(63, 12), addr) << 1; | |
| temp_addr_b |= FIELD_GET(BIT_ULL(8), addr); | |
| temp_addr_b /= denorm_ctx->mod_value; | |
| temp_addr_b <<= 11; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return 0; | |
| } | |
| return denorm_ctx->base_denorm_addr | temp_addr_a | temp_addr_b; | |
| } | |
| static void recalculate_hashed_bits_df4p5_np2(struct addr_ctx *ctx, | |
| struct df4p5_denorm_ctx *denorm_ctx) | |
| { | |
| bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G, hash_ctl_1T, hashed_bit; | |
| if (!denorm_ctx->rehash_vector) | |
| return; | |
| hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl); | |
| hash_ctl_2M = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl); | |
| hash_ctl_1G = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl); | |
| hash_ctl_1T = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl); | |
| if (denorm_ctx->rehash_vector & BIT_ULL(8)) { | |
| hashed_bit = FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(14), denorm_ctx->current_spa); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(16), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(21), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(30), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(40), denorm_ctx->current_spa) & hash_ctl_1T; | |
| if (FIELD_GET(BIT_ULL(8), denorm_ctx->current_spa) != hashed_bit) | |
| denorm_ctx->current_spa ^= BIT_ULL(8); | |
| } | |
| if (denorm_ctx->rehash_vector & BIT_ULL(9)) { | |
| hashed_bit = FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(17), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(22), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(31), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(41), denorm_ctx->current_spa) & hash_ctl_1T; | |
| if (FIELD_GET(BIT_ULL(9), denorm_ctx->current_spa) != hashed_bit) | |
| denorm_ctx->current_spa ^= BIT_ULL(9); | |
| } | |
| if (denorm_ctx->rehash_vector & BIT_ULL(12)) { | |
| hashed_bit = FIELD_GET(BIT_ULL(12), denorm_ctx->current_spa); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(18), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(23), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(32), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(42), denorm_ctx->current_spa) & hash_ctl_1T; | |
| if (FIELD_GET(BIT_ULL(12), denorm_ctx->current_spa) != hashed_bit) | |
| denorm_ctx->current_spa ^= BIT_ULL(12); | |
| } | |
| if (denorm_ctx->rehash_vector & BIT_ULL(13)) { | |
| hashed_bit = FIELD_GET(BIT_ULL(13), denorm_ctx->current_spa); | |
| hashed_bit ^= FIELD_GET(BIT_ULL(19), denorm_ctx->current_spa) & hash_ctl_64k; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(24), denorm_ctx->current_spa) & hash_ctl_2M; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(33), denorm_ctx->current_spa) & hash_ctl_1G; | |
| hashed_bit ^= FIELD_GET(BIT_ULL(43), denorm_ctx->current_spa) & hash_ctl_1T; | |
| if (FIELD_GET(BIT_ULL(13), denorm_ctx->current_spa) != hashed_bit) | |
| denorm_ctx->current_spa ^= BIT_ULL(13); | |
| } | |
| } | |
| static bool match_logical_coh_st_fabric_id(struct addr_ctx *ctx, | |
| struct df4p5_denorm_ctx *denorm_ctx) | |
| { | |
| /* | |
| * The logical CS fabric ID of the permutation must be calculated from the | |
| * current SPA with the base and with the MMIO hole. | |
| */ | |
| u16 id = get_logical_coh_st_fabric_id_for_current_spa(ctx, denorm_ctx); | |
| atl_debug(ctx, "Checking calculated logical coherent station fabric id:\n"); | |
| atl_debug(ctx, " calculated fabric id = 0x%x\n", id); | |
| atl_debug(ctx, " expected fabric id = 0x%x\n", denorm_ctx->coh_st_fabric_id); | |
| return denorm_ctx->coh_st_fabric_id == id; | |
| } | |
| static bool match_norm_addr(struct addr_ctx *ctx, struct df4p5_denorm_ctx *denorm_ctx) | |
| { | |
| u64 addr = remove_base_and_hole(ctx, denorm_ctx->current_spa); | |
| /* | |
| * The normalized address must be calculated with the current SPA without | |
| * the base and without the MMIO hole. | |
| */ | |
| addr = normalize_addr_df4p5_np2(ctx, denorm_ctx, addr); | |
| atl_debug(ctx, "Checking calculated normalized address:\n"); | |
| atl_debug(ctx, " calculated normalized addr = 0x%016llx\n", addr); | |
| atl_debug(ctx, " expected normalized addr = 0x%016llx\n", ctx->ret_addr); | |
| return addr == ctx->ret_addr; | |
| } | |
| static int check_permutations(struct addr_ctx *ctx, struct df4p5_denorm_ctx *denorm_ctx) | |
| { | |
| u64 test_perm, temp_addr, denorm_addr, num_perms; | |
| unsigned int dropped_remainder; | |
| denorm_ctx->div_addr *= denorm_ctx->mod_value; | |
| /* | |
| * The high order bits of num_permutations represent the permutations | |
| * of the dropped remainder. This will be either 0-3 or 0-5 depending | |
| * on the interleave mode. The low order bits represent the | |
| * permutations of other "lost" bits which will be any combination of | |
| * 1, 2, or 3 bits depending on the interleave mode. | |
| */ | |
| num_perms = denorm_ctx->mod_value << denorm_ctx->perm_shift; | |
| for (test_perm = 0; test_perm < num_perms; test_perm++) { | |
| denorm_addr = denorm_ctx->base_denorm_addr; | |
| dropped_remainder = test_perm >> denorm_ctx->perm_shift; | |
| temp_addr = denorm_ctx->div_addr + dropped_remainder; | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| denorm_addr |= temp_addr << 14; | |
| break; | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| denorm_addr |= temp_addr << 13; | |
| break; | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| denorm_addr |= temp_addr << 12; | |
| break; | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), temp_addr) << 9; | |
| denorm_addr |= FIELD_GET(GENMASK_ULL(63, 1), temp_addr) << 12; | |
| break; | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| denorm_addr |= FIELD_GET(GENMASK_ULL(1, 0), temp_addr) << 8; | |
| denorm_addr |= FIELD_GET(GENMASK_ULL(63, 2), (temp_addr)) << 12; | |
| break; | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), temp_addr) << 8; | |
| denorm_addr |= FIELD_GET(GENMASK_ULL(63, 1), temp_addr) << 12; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return -EINVAL; | |
| } | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), test_perm) << 8; | |
| denorm_addr |= FIELD_GET(BIT_ULL(1), test_perm) << 9; | |
| denorm_addr |= FIELD_GET(BIT_ULL(2), test_perm) << 12; | |
| break; | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), test_perm) << 8; | |
| denorm_addr |= FIELD_GET(BIT_ULL(1), test_perm) << 12; | |
| denorm_addr |= FIELD_GET(BIT_ULL(2), test_perm) << 13; | |
| break; | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), test_perm) << 8; | |
| denorm_addr |= FIELD_GET(BIT_ULL(1), test_perm) << 12; | |
| break; | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), test_perm) << 8; | |
| denorm_addr |= FIELD_GET(BIT_ULL(1), test_perm) << 9; | |
| break; | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| denorm_addr |= FIELD_GET(BIT_ULL(0), test_perm) << 8; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return -EINVAL; | |
| } | |
| denorm_ctx->current_spa = add_base_and_hole(ctx, denorm_addr); | |
| recalculate_hashed_bits_df4p5_np2(ctx, denorm_ctx); | |
| atl_debug(ctx, "Checking potential system physical address 0x%016llx\n", | |
| denorm_ctx->current_spa); | |
| if (!match_logical_coh_st_fabric_id(ctx, denorm_ctx)) | |
| continue; | |
| if (!match_norm_addr(ctx, denorm_ctx)) | |
| continue; | |
| if (denorm_ctx->resolved_spa == INVALID_SPA || | |
| denorm_ctx->current_spa > denorm_ctx->resolved_spa) | |
| denorm_ctx->resolved_spa = denorm_ctx->current_spa; | |
| } | |
| if (denorm_ctx->resolved_spa == INVALID_SPA) { | |
| atl_debug(ctx, "Failed to find valid SPA for normalized address 0x%016llx\n", | |
| ctx->ret_addr); | |
| return -EINVAL; | |
| } | |
| /* Return the resolved SPA without the base, without the MMIO hole */ | |
| ctx->ret_addr = remove_base_and_hole(ctx, denorm_ctx->resolved_spa); | |
| return 0; | |
| } | |
| static int init_df4p5_denorm_ctx(struct addr_ctx *ctx, struct df4p5_denorm_ctx *denorm_ctx) | |
| { | |
| denorm_ctx->current_spa = INVALID_SPA; | |
| denorm_ctx->resolved_spa = INVALID_SPA; | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| denorm_ctx->perm_shift = 3; | |
| denorm_ctx->rehash_vector = BIT(8) | BIT(9) | BIT(12); | |
| break; | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| denorm_ctx->perm_shift = 3; | |
| denorm_ctx->rehash_vector = BIT(8) | BIT(12) | BIT(13); | |
| break; | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| denorm_ctx->perm_shift = 2; | |
| denorm_ctx->rehash_vector = BIT(8); | |
| break; | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| denorm_ctx->perm_shift = 2; | |
| denorm_ctx->rehash_vector = BIT(8) | BIT(12); | |
| break; | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| denorm_ctx->perm_shift = 1; | |
| denorm_ctx->rehash_vector = BIT(8); | |
| break; | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| denorm_ctx->perm_shift = 2; | |
| denorm_ctx->rehash_vector = 0; | |
| break; | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| denorm_ctx->perm_shift = 1; | |
| denorm_ctx->rehash_vector = 0; | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return -EINVAL; | |
| } | |
| denorm_ctx->base_denorm_addr = FIELD_GET(GENMASK_ULL(7, 0), ctx->ret_addr); | |
| switch (ctx->map.intlv_mode) { | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| denorm_ctx->base_denorm_addr |= FIELD_GET(GENMASK_ULL(9, 8), ctx->ret_addr) << 10; | |
| denorm_ctx->div_addr = FIELD_GET(GENMASK_ULL(63, 10), ctx->ret_addr); | |
| break; | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| denorm_ctx->base_denorm_addr |= FIELD_GET(GENMASK_ULL(10, 8), ctx->ret_addr) << 9; | |
| denorm_ctx->div_addr = FIELD_GET(GENMASK_ULL(63, 11), ctx->ret_addr); | |
| break; | |
| default: | |
| atl_debug_on_bad_intlv_mode(ctx); | |
| return -EINVAL; | |
| } | |
| if (ctx->map.num_intlv_chan % 3 == 0) | |
| denorm_ctx->mod_value = 3; | |
| else | |
| denorm_ctx->mod_value = 5; | |
| denorm_ctx->coh_st_fabric_id = get_logical_coh_st_fabric_id(ctx) - get_dst_fabric_id(ctx); | |
| atl_debug(ctx, "Initialized df4p5_denorm_ctx:"); | |
| atl_debug(ctx, " mod_value = %d", denorm_ctx->mod_value); | |
| atl_debug(ctx, " perm_shift = %d", denorm_ctx->perm_shift); | |
| atl_debug(ctx, " rehash_vector = 0x%x", denorm_ctx->rehash_vector); | |
| atl_debug(ctx, " base_denorm_addr = 0x%016llx", denorm_ctx->base_denorm_addr); | |
| atl_debug(ctx, " div_addr = 0x%016llx", denorm_ctx->div_addr); | |
| atl_debug(ctx, " coh_st_fabric_id = 0x%x", denorm_ctx->coh_st_fabric_id); | |
| return 0; | |
| } | |
| /* | |
| * For DF 4.5, parts of the physical address can be directly pulled from the | |
| * normalized address. The exact bits will differ between interleave modes, but | |
| * using NPS0_24CHAN_1K_HASH as an example, the normalized address consists of | |
| * bits [63:13] (divided by 3), bits [11:10], and bits [7:0] of the system | |
| * physical address. | |
| * | |
| * In this case, there is no way to reconstruct the missing bits (bits 8, 9, | |
| * and 12) from the normalized address. Additionally, when bits [63:13] are | |
| * divided by 3, the remainder is dropped. Determine the proper combination of | |
| * "lost" bits and dropped remainder by iterating through each possible | |
| * permutation of these bits and then normalizing the generated system physical | |
| * addresses. If the normalized address matches the address we are trying to | |
| * translate, then we have found the correct permutation of bits. | |
| */ | |
| static int denorm_addr_df4p5_np2(struct addr_ctx *ctx) | |
| { | |
| struct df4p5_denorm_ctx denorm_ctx; | |
| int ret = 0; | |
| memset(&denorm_ctx, 0, sizeof(denorm_ctx)); | |
| atl_debug(ctx, "Denormalizing DF 4.5 normalized address 0x%016llx", ctx->ret_addr); | |
| ret = init_df4p5_denorm_ctx(ctx, &denorm_ctx); | |
| if (ret) | |
| return ret; | |
| return check_permutations(ctx, &denorm_ctx); | |
| } | |
| int denormalize_address(struct addr_ctx *ctx) | |
| { | |
| switch (ctx->map.intlv_mode) { | |
| case NONE: | |
| return 0; | |
| case DF4_NPS4_3CHAN_HASH: | |
| case DF4_NPS2_6CHAN_HASH: | |
| case DF4_NPS1_12CHAN_HASH: | |
| case DF4_NPS2_5CHAN_HASH: | |
| case DF4_NPS1_10CHAN_HASH: | |
| return denorm_addr_df4_np2(ctx); | |
| case DF4p5_NPS0_24CHAN_1K_HASH: | |
| case DF4p5_NPS4_3CHAN_1K_HASH: | |
| case DF4p5_NPS2_6CHAN_1K_HASH: | |
| case DF4p5_NPS1_12CHAN_1K_HASH: | |
| case DF4p5_NPS2_5CHAN_1K_HASH: | |
| case DF4p5_NPS1_10CHAN_1K_HASH: | |
| case DF4p5_NPS4_3CHAN_2K_HASH: | |
| case DF4p5_NPS2_6CHAN_2K_HASH: | |
| case DF4p5_NPS1_12CHAN_2K_HASH: | |
| case DF4p5_NPS0_24CHAN_2K_HASH: | |
| case DF4p5_NPS2_5CHAN_2K_HASH: | |
| case DF4p5_NPS1_10CHAN_2K_HASH: | |
| return denorm_addr_df4p5_np2(ctx); | |
| case DF3_6CHAN: | |
| return denorm_addr_df3_6chan(ctx); | |
| default: | |
| return denorm_addr_common(ctx); | |
| } | |
| } |