// Copyright lowRISC contributors.
// Copyright 2018 ETH Zurich and University of Bologna, see also CREDITS.md.
// Licensed under the Apache License, Version 2.0, see LICENSE for details.
// SPDX-License-Identifier: Apache-2.0
/**
* Load Store Unit
*
* Load Store Unit, used to eliminate multiple access during processor stalls,
* and to align bytes and halfwords.
*/
`include "prim_assert.sv"
`include "dv_fcov_macros.svh"
module [docs]ibex_load_store_unit #(
parameter bit MemECC = 1'b0,
parameter int unsigned MemDataWidth = MemECC ? 32 + 7 : 32
) (
input logic clk_i,
input logic rst_ni,
// data interface
output logic data_req_o,
input logic data_gnt_i,
input logic data_rvalid_i,
input logic data_bus_err_i,
input logic data_pmp_err_i,
output logic [31:0] data_addr_o,
output logic data_we_o,
output logic [3:0] data_be_o,
output logic [MemDataWidth-1:0] data_wdata_o,
input logic [MemDataWidth-1:0] data_rdata_i,
// signals to/from ID/EX stage
input logic lsu_we_i, // write enable -> from ID/EX
input logic [1:0] lsu_type_i, // data type: word, half word, byte -> from ID/EX
input logic [31:0] lsu_wdata_i, // data to write to memory -> from ID/EX
input logic lsu_sign_ext_i, // sign extension -> from ID/EX
output logic [31:0] lsu_rdata_o, // requested data -> to ID/EX
output logic lsu_rdata_valid_o,
input logic lsu_req_i, // data request -> from ID/EX
input logic [31:0] adder_result_ex_i, // address computed in ALU -> from ID/EX
output logic addr_incr_req_o, // request address increment for
// misaligned accesses -> to ID/EX
output logic [31:0] addr_last_o, // address of last transaction -> to controller
// -> mtval
// -> AGU for misaligned accesses
output logic lsu_req_done_o, // Signals that data request is complete
// (only need to await final data
// response) -> to ID/EX
output logic lsu_resp_valid_o, // LSU has response from transaction -> to ID/EX
// exception signals
output logic load_err_o,
output logic load_resp_intg_err_o,
output logic store_err_o,
output logic store_resp_intg_err_o,
output logic busy_o,
output logic perf_load_o,
output logic perf_store_o
);
logic [31:0] data_addr;
logic [31:0] data_addr_w_aligned;
logic [31:0] addr_last_q, addr_last_d;
logic addr_update;
logic ctrl_update;
logic rdata_update;
logic [31:8] rdata_q;
logic [1:0] rdata_offset_q;
logic [1:0] data_type_q;
logic data_sign_ext_q;
logic data_we_q;
logic [1:0] data_offset; // mux control for data to be written to memory
logic [3:0] data_be;
logic [31:0] data_wdata;
logic [31:0] data_rdata_ext;
logic [31:0] rdata_w_ext; // word realignment for misaligned loads
logic [31:0] rdata_h_ext; // sign extension for half words
logic [31:0] rdata_b_ext; // sign extension for bytes
logic split_misaligned_access;
logic handle_misaligned_q, handle_misaligned_d; // high after receiving grant for first
// part of a misaligned access
logic pmp_err_q, pmp_err_d;
logic lsu_err_q, lsu_err_d;
logic data_intg_err, data_or_pmp_err;
typedef enum logic [2:0] {
IDLE, WAIT_GNT_MIS, WAIT_RVALID_MIS, WAIT_GNT,
WAIT_RVALID_MIS_GNTS_DONE
} ls_fsm_e;
ls_fsm_e ls_fsm_cs, ls_fsm_ns;
assign data_addr = adder_result_ex_i;
assign data_offset = data_addr[1:0];
///////////////////
// BE generation //
///////////////////
always_comb[docs] begin
unique case (lsu_type_i) // Data type 00 Word, 01 Half word, 11,10 byte
2'b00: begin // Writing a word
if (!handle_misaligned_q) begin // first part of potentially misaligned transaction
unique case (data_offset)
2'b00: data_be = 4'b1111;
2'b01: data_be = 4'b1110;
2'b10: data_be = 4'b1100;
2'b11: data_be = 4'b1000;
default: data_be = 4'b1111;
endcase // case (data_offset)
end else begin // second part of misaligned transaction
unique case (data_offset)
2'b00: data_be = 4'b0000; // this is not used, but included for completeness
2'b01: data_be = 4'b0001;
2'b10: data_be = 4'b0011;
2'b11: data_be = 4'b0111;
default: data_be = 4'b1111;
endcase // case (data_offset)
end
end
2'b01: begin // Writing a half word
if (!handle_misaligned_q) begin // first part of potentially misaligned transaction
unique case (data_offset)
2'b00: data_be = 4'b0011;
2'b01: data_be = 4'b0110;
2'b10: data_be = 4'b1100;
2'b11: data_be = 4'b1000;
default: data_be = 4'b1111;
endcase // case (data_offset)
end else begin // second part of misaligned transaction
data_be = 4'b0001;
end
end
2'b10,
2'b11: begin // Writing a byte
unique case (data_offset)
2'b00: data_be = 4'b0001;
2'b01: data_be = 4'b0010;
2'b10: data_be = 4'b0100;
2'b11: data_be = 4'b1000;
default: data_be = 4'b1111;
endcase // case (data_offset)
end
default: data_be = 4'b1111;
endcase // case (lsu_type_i)
end
/////////////////////
// WData alignment //
/////////////////////
// prepare data to be written to the memory
// we handle misaligned accesses, half word and byte accesses here
always_comb[docs] begin
unique case (data_offset)
2'b00: data_wdata = lsu_wdata_i[31:0];
2'b01: data_wdata = {lsu_wdata_i[23:0], lsu_wdata_i[31:24]};
2'b10: data_wdata = {lsu_wdata_i[15:0], lsu_wdata_i[31:16]};
2'b11: data_wdata = {lsu_wdata_i[ 7:0], lsu_wdata_i[31: 8]};
default: data_wdata = lsu_wdata_i[31:0];
endcase // case (data_offset)
end
/////////////////////
// RData alignment //
/////////////////////
// register for unaligned rdata
always_ff[docs] @(posedge clk_i or negedge rst_ni) begin
if (!rst_ni) begin
rdata_q <= '0;
end else if (rdata_update) begin
rdata_q <= data_rdata_i[31:8];
end
end
// registers for transaction control
always_ff[docs] @(posedge clk_i or negedge rst_ni) begin
if (!rst_ni) begin
rdata_offset_q <= 2'h0;
data_type_q <= 2'h0;
data_sign_ext_q <= 1'b0;
data_we_q <= 1'b0;
end else if (ctrl_update) begin
rdata_offset_q <= data_offset;
data_type_q <= lsu_type_i;
data_sign_ext_q <= lsu_sign_ext_i;
data_we_q <= lsu_we_i;
end
end
// Store last address for mtval + AGU for misaligned transactions. Do not update in case of
// errors, mtval needs the (first) failing address. Where an aligned access or the first half of
// a misaligned access sees an error provide the calculated access address. For the second half of
// a misaligned access provide the word aligned address of the second half.
assign addr_last_d = addr_incr_req_o ? data_addr_w_aligned : data_addr;
always_ff @(posedge clk_i or negedge rst_ni) begin
if (!rst_ni) begin
addr_last_q <= '0;
end else if (addr_update) begin
addr_last_q <= addr_last_d;
end
end
// take care of misaligned words
always_comb[docs] begin
unique case (rdata_offset_q)
2'b00: rdata_w_ext = data_rdata_i[31:0];
2'b01: rdata_w_ext = {data_rdata_i[ 7:0], rdata_q[31:8]};
2'b10: rdata_w_ext = {data_rdata_i[15:0], rdata_q[31:16]};
2'b11: rdata_w_ext = {data_rdata_i[23:0], rdata_q[31:24]};
default: rdata_w_ext = data_rdata_i[31:0];
endcase
end
////////////////////
// Sign extension //
////////////////////
// sign extension for half words
always_comb[docs] begin
unique case (rdata_offset_q)
2'b00: begin
if (!data_sign_ext_q) begin
rdata_h_ext = {16'h0000, data_rdata_i[15:0]};
end else begin
rdata_h_ext = {{16{data_rdata_i[15]}}, data_rdata_i[15:0]};
end
end
2'b01: begin
if (!data_sign_ext_q) begin
rdata_h_ext = {16'h0000, data_rdata_i[23:8]};
end else begin
rdata_h_ext = {{16{data_rdata_i[23]}}, data_rdata_i[23:8]};
end
end
2'b10: begin
if (!data_sign_ext_q) begin
rdata_h_ext = {16'h0000, data_rdata_i[31:16]};
end else begin
rdata_h_ext = {{16{data_rdata_i[31]}}, data_rdata_i[31:16]};
end
end
2'b11: begin
if (!data_sign_ext_q) begin
rdata_h_ext = {16'h0000, data_rdata_i[7:0], rdata_q[31:24]};
end else begin
rdata_h_ext = {{16{data_rdata_i[7]}}, data_rdata_i[7:0], rdata_q[31:24]};
end
end
default: rdata_h_ext = {16'h0000, data_rdata_i[15:0]};
endcase // case (rdata_offset_q)
end
// sign extension for bytes
always_comb[docs] begin
unique case (rdata_offset_q)
2'b00: begin
if (!data_sign_ext_q) begin
rdata_b_ext = {24'h00_0000, data_rdata_i[7:0]};
end else begin
rdata_b_ext = {{24{data_rdata_i[7]}}, data_rdata_i[7:0]};
end
end
2'b01: begin
if (!data_sign_ext_q) begin
rdata_b_ext = {24'h00_0000, data_rdata_i[15:8]};
end else begin
rdata_b_ext = {{24{data_rdata_i[15]}}, data_rdata_i[15:8]};
end
end
2'b10: begin
if (!data_sign_ext_q) begin
rdata_b_ext = {24'h00_0000, data_rdata_i[23:16]};
end else begin
rdata_b_ext = {{24{data_rdata_i[23]}}, data_rdata_i[23:16]};
end
end
2'b11: begin
if (!data_sign_ext_q) begin
rdata_b_ext = {24'h00_0000, data_rdata_i[31:24]};
end else begin
rdata_b_ext = {{24{data_rdata_i[31]}}, data_rdata_i[31:24]};
end
end
default: rdata_b_ext = {24'h00_0000, data_rdata_i[7:0]};
endcase // case (rdata_offset_q)
end
// select word, half word or byte sign extended version
always_comb[docs] begin
unique case (data_type_q)
2'b00: data_rdata_ext = rdata_w_ext;
2'b01: data_rdata_ext = rdata_h_ext;
2'b10,2'b11: data_rdata_ext = rdata_b_ext;
default: data_rdata_ext = rdata_w_ext;
endcase // case (data_type_q)
end
///////////////////////////////
// Read data integrity check //
///////////////////////////////
// SEC_CM: BUS.INTEGRITY
if (MemECC) begin : g_mem_rdata_ecc
logic [1:0] ecc_err;
logic [MemDataWidth-1:0] data_rdata_buf;
prim_buf #(.Width(MemDataWidth)) u_prim_buf_instr_rdata (
.in_i (data_rdata_i),
.out_o(data_rdata_buf)
);
prim_secded_inv_39_32_dec u_data_intg_dec (
.data_i (data_rdata_buf),
.data_o (),
.syndrome_o (),
.err_o (ecc_err)
);
// Don't care if error is correctable or not, they're all treated the same
assign data_intg_err = |ecc_err;
end else begin : g_no_mem_data_ecc
assign data_intg_err = 1'b0;
end
/////////////
// LSU FSM //
/////////////
// check for misaligned accesses that need to be split into two word-aligned accesses
assign split_misaligned_access =
((lsu_type_i == 2'b00) && (data_offset != 2'b00)) || // misaligned word access
((lsu_type_i == 2'b01) && (data_offset == 2'b11)); // misaligned half-word access
// FSM
always_comb[docs] begin
ls_fsm_ns = ls_fsm_cs;
data_req_o = 1'b0;
addr_incr_req_o = 1'b0;
handle_misaligned_d = handle_misaligned_q;
pmp_err_d = pmp_err_q;
lsu_err_d = lsu_err_q;
addr_update = 1'b0;
ctrl_update = 1'b0;
rdata_update = 1'b0;
perf_load_o = 1'b0;
perf_store_o = 1'b0;
unique case (ls_fsm_cs)
IDLE: begin
pmp_err_d = 1'b0;
if (lsu_req_i) begin
data_req_o = 1'b1;
pmp_err_d = data_pmp_err_i;
lsu_err_d = 1'b0;
perf_load_o = ~lsu_we_i;
perf_store_o = lsu_we_i;
if (data_gnt_i) begin
ctrl_update = 1'b1;
addr_update = 1'b1;
handle_misaligned_d = split_misaligned_access;
ls_fsm_ns = split_misaligned_access ? WAIT_RVALID_MIS : IDLE;
end else begin
ls_fsm_ns = split_misaligned_access ? WAIT_GNT_MIS : WAIT_GNT;
end
end
end
WAIT_GNT_MIS: begin
data_req_o = 1'b1;
// data_pmp_err_i is valid during the address phase of a request. An error will block the
// external request and so a data_gnt_i might never be signalled. The registered version
// pmp_err_q is only updated for new address phases and so can be used in WAIT_GNT* and
// WAIT_RVALID* states
if (data_gnt_i || pmp_err_q) begin
addr_update = 1'b1;
ctrl_update = 1'b1;
handle_misaligned_d = 1'b1;
ls_fsm_ns = WAIT_RVALID_MIS;
end
end
WAIT_RVALID_MIS: begin
// push out second request
data_req_o = 1'b1;
// tell ID/EX stage to update the address
addr_incr_req_o = 1'b1;
// first part rvalid is received, or gets a PMP error
if (data_rvalid_i || pmp_err_q) begin
// Update the PMP error for the second part
pmp_err_d = data_pmp_err_i;
// Record the error status of the first part
lsu_err_d = data_bus_err_i | pmp_err_q;
// Capture the first rdata for loads
rdata_update = ~data_we_q;
// If already granted, wait for second rvalid
ls_fsm_ns = data_gnt_i ? IDLE : WAIT_GNT;
// Update the address for the second part, if no error
addr_update = data_gnt_i & ~(data_bus_err_i | pmp_err_q);
// clear handle_misaligned if second request is granted
handle_misaligned_d = ~data_gnt_i;
end else begin
// first part rvalid is NOT received
if (data_gnt_i) begin
// second grant is received
ls_fsm_ns = WAIT_RVALID_MIS_GNTS_DONE;
handle_misaligned_d = 1'b0;
end
end
end
WAIT_GNT: begin
// tell ID/EX stage to update the address
addr_incr_req_o = handle_misaligned_q;
data_req_o = 1'b1;
if (data_gnt_i || pmp_err_q) begin
ctrl_update = 1'b1;
// Update the address, unless there was an error
addr_update = ~lsu_err_q;
ls_fsm_ns = IDLE;
handle_misaligned_d = 1'b0;
end
end
WAIT_RVALID_MIS_GNTS_DONE: begin
// tell ID/EX stage to update the address (to make sure the
// second address can be captured correctly for mtval and PMP checking)
addr_incr_req_o = 1'b1;
// Wait for the first rvalid, second request is already granted
if (data_rvalid_i) begin
// Update the pmp error for the second part
pmp_err_d = data_pmp_err_i;
// The first part cannot see a PMP error in this state
lsu_err_d = data_bus_err_i;
// Now we can update the address for the second part if no error
addr_update = ~data_bus_err_i;
// Capture the first rdata for loads
rdata_update = ~data_we_q;
// Wait for second rvalid
ls_fsm_ns = IDLE;
end
end
default: begin
ls_fsm_ns = IDLE;
end
endcase
end
assign lsu_req_done_o = (lsu_req_i | (ls_fsm_cs != IDLE)) & (ls_fsm_ns == IDLE);
// registers for FSM
always_ff[docs] @(posedge clk_i or negedge rst_ni) begin
if (!rst_ni) begin
ls_fsm_cs <= IDLE;
handle_misaligned_q <= '0;
pmp_err_q <= '0;
lsu_err_q <= '0;
end else begin
ls_fsm_cs <= ls_fsm_ns;
handle_misaligned_q <= handle_misaligned_d;
pmp_err_q <= pmp_err_d;
lsu_err_q <= lsu_err_d;
end
end
/////////////
// Outputs //
/////////////
assign data_or_pmp_err = lsu_err_q | data_bus_err_i | pmp_err_q;
assign lsu_resp_valid_o = (data_rvalid_i | pmp_err_q) & (ls_fsm_cs == IDLE);
assign lsu_rdata_valid_o =
(ls_fsm_cs == IDLE) & data_rvalid_i & ~data_or_pmp_err & ~data_we_q & ~data_intg_err;
// output to register file
assign lsu_rdata_o = data_rdata_ext;
// output data address must be word aligned
assign data_addr_w_aligned = {data_addr[31:2], 2'b00};
// output to data interface
assign data_addr_o = data_addr_w_aligned;
assign data_we_o = lsu_we_i;
assign data_be_o = data_be;
/////////////////////////////////////
// Write data integrity generation //
/////////////////////////////////////
// SEC_CM: BUS.INTEGRITY
if (MemECC) begin : g_mem_wdata_ecc
prim_secded_inv_39_32_enc u_data_gen (
.data_i (data_wdata),
.data_o (data_wdata_o)
);
end else begin : g_no_mem_wdata_ecc
assign data_wdata_o = data_wdata;
end
// output to ID stage: mtval + AGU for misaligned transactions
assign addr_last_o = addr_last_q;
// Signal a load or store error depending on the transaction type outstanding
assign load_err_o = data_or_pmp_err & ~data_we_q & lsu_resp_valid_o;
assign store_err_o = data_or_pmp_err & data_we_q & lsu_resp_valid_o;
// Integrity errors are their own category for timing reasons. load_err_o is factored directly
// into data_req_o to enable synchronous exception on load errors without performance loss (An
// upcoming load cannot request until the current load has seen its response, so the earliest
// point the new request can be sent is the same cycle the response is seen). If load_err_o isn't
// factored into data_req_o there would have to be a stall cycle between all back to back loads.
// The data_intg_err signal is generated combinatorially from the incoming data_rdata_i. Were it
// to be factored into load_err_o there would be a feedthrough path from data_rdata_i to
// data_req_o which is undesirable.
assign load_resp_intg_err_o = data_intg_err & data_rvalid_i & ~data_we_q;
assign store_resp_intg_err_o = data_intg_err & data_rvalid_i & data_we_q;
assign busy_o = (ls_fsm_cs != IDLE);
//////////
// FCOV //
//////////
`ifndef DV_FCOV_DISABLE
// Set when awaiting the response for the second half of a misaligned access
logic fcov_mis_2_en_d, fcov_mis_2_en_q;
// fcov_mis_rvalid_1: Set when the response is received to the first half of a misaligned access,
// fcov_mis_rvalid_2: Set when response is received for the second half
logic fcov_mis_rvalid_1, fcov_mis_rvalid_2;
// Set when the first half of a misaligned access saw a bus errror
logic fcov_mis_bus_err_1_d, fcov_mis_bus_err_1_q;
assign fcov_mis_rvalid_1 = ls_fsm_cs inside {WAIT_RVALID_MIS, WAIT_RVALID_MIS_GNTS_DONE} &&
data_rvalid_i;
assign fcov_mis_rvalid_2 = ls_fsm_cs inside {IDLE} && fcov_mis_2_en_q && data_rvalid_i;
assign fcov_mis_2_en_d = fcov_mis_rvalid_2 ? 1'b0 : // clr
fcov_mis_rvalid_1 ? 1'b1 : // set
fcov_mis_2_en_q ;
assign fcov_mis_bus_err_1_d = fcov_mis_rvalid_2 ? 1'b0 : // clr
fcov_mis_rvalid_1 && data_bus_err_i ? 1'b1 : // set
fcov_mis_bus_err_1_q ;
always_ff @(posedge clk_i or negedge rst_ni) begin
if (!rst_ni) begin
fcov_mis_2_en_q <= 1'b0;
fcov_mis_bus_err_1_q <= 1'b0;
end else begin
fcov_mis_2_en_q <= fcov_mis_2_en_d;
fcov_mis_bus_err_1_q <= fcov_mis_bus_err_1_d;
end
end
`endif
`DV_FCOV_SIGNAL(logic, ls_error_exception, (load_err_o | store_err_o) & ~pmp_err_q)
`DV_FCOV_SIGNAL(logic, ls_pmp_exception, (load_err_o | store_err_o) & pmp_err_q)
`DV_FCOV_SIGNAL(logic, ls_first_req, lsu_req_i & (ls_fsm_cs == IDLE))
`DV_FCOV_SIGNAL(logic, ls_second_req,
(ls_fsm_cs inside {WAIT_RVALID_MIS}) & data_req_o & addr_incr_req_o)
`DV_FCOV_SIGNAL(logic, ls_mis_pmp_err_1,
(ls_fsm_cs inside {WAIT_RVALID_MIS, WAIT_GNT_MIS}) && pmp_err_q)
`DV_FCOV_SIGNAL(logic, ls_mis_pmp_err_2,
(ls_fsm_cs inside {WAIT_RVALID_MIS, WAIT_RVALID_MIS_GNTS_DONE}) && data_pmp_err_i)
////////////////
// Assertions //
////////////////
// Selectors must be known/valid.
`ASSERT(IbexDataTypeKnown, (lsu_req_i | busy_o) |-> !$isunknown(lsu_type_i))
`ASSERT(IbexDataOffsetKnown, (lsu_req_i | busy_o) |-> !$isunknown(data_offset))
`ASSERT_KNOWN(IbexRDataOffsetQKnown, rdata_offset_q)
`ASSERT_KNOWN(IbexDataTypeQKnown, data_type_q)
`ASSERT(IbexLsuStateValid, ls_fsm_cs inside {
IDLE, WAIT_GNT_MIS, WAIT_RVALID_MIS, WAIT_GNT,
WAIT_RVALID_MIS_GNTS_DONE})
// Address must not contain X when request is sent.
`ASSERT(IbexDataAddrUnknown, data_req_o |-> !$isunknown(data_addr_o))
// Address must be word aligned when request is sent.
`ASSERT(IbexDataAddrUnaligned, data_req_o |-> (data_addr_o[1:0] == 2'b00))
endmodule