diff --git a/src/main.rs b/src/main.rs index 0acf8e0..fecbfad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -111,6 +111,8 @@ struct Config { dc: bool, /// Use the io_uring-based blocking TX/RX loop instead of AF_PACKET poll. iouring: bool, + /// RFM device path override (e.g. /dev/rfm2g1) for multi-card hosts. + rfm_dev: Option, } fn parse_args() -> Result { @@ -131,6 +133,7 @@ fn parse_args() -> Result { spin: false, dc: false, iouring: false, + rfm_dev: None, }; let mut args = std::env::args().skip(1).peekable(); if let Some(a) = args.peek() { @@ -166,6 +169,10 @@ fn parse_args() -> Result { "--spin" => cfg.spin = true, "--dc" => cfg.dc = true, "--iouring" => cfg.iouring = true, + "--rfm-dev" => { + let n: u32 = val("--rfm-dev")?.parse().map_err(|_| "bad --rfm-dev (use card index, e.g. 0 or 1)")?; + cfg.rfm_dev = Some(format!("/dev/rfm2g{}", n)); + } "--el2202" => cfg.el2202 = true, "--el2202-dual" => { cfg.el2202 = true; @@ -954,8 +961,7 @@ async fn run(cfg: &Config) -> Result<(), Box> { // RFM bridge context (only opened in rfm-bridge mode). let mut rfm_ctx: Option = if cfg.mode == Mode::RfmBridge { - println!("rfm-bridge: opening PCI-5565 via VFIO..."); - let r = el6695_rt::rfm::Rfm::open()?; + let r = el6695_rt::rfm::Rfm::open_path(cfg.rfm_dev.as_deref())?; println!( "rfm-bridge: BAR2 reflective mem = {} MB, node_id={}", r.mem.len() / (1024 * 1024), @@ -1102,7 +1108,7 @@ async fn run(cfg: &Config) -> Result<(), Box> { bridge_req_seq = bridge_req_seq.wrapping_add(1); unsafe { let mb = el6695_rt::rfm::Mailbox::at(mem_ptr, el6695_rt::rfm::MB_REQ_OFF); - mb.payload[..n].copy_from_slice(&rx_buf[..n]); + el6695_rt::rfm::dev_write(mb.payload.as_mut_ptr(), &rx_buf[..n]); mb.len = n as u32; mb.ts_ns = ts; el6695_rt::rfm::barrier(); @@ -1122,7 +1128,7 @@ async fn run(cfg: &Config) -> Result<(), Box> { && rep.len as usize >= SM_RX_PLD_LEN.min(el6695_rt::rfm::MB_MAX_PAYLOAD) { let m = SM_RX_PLD_LEN.min(el6695_rt::rfm::MB_MAX_PAYLOAD); - tx_buf[..m].copy_from_slice(&rep.payload[..m]); + el6695_rt::rfm::dev_read(&mut tx_buf[..m], rep.payload.as_ptr()); bridge_rtt_st.push(el6695_rt::rfm::now_ns().saturating_sub(rep.ts_ns) as i64); got = true; break; @@ -1491,15 +1497,30 @@ async fn run(cfg: &Config) -> Result<(), Box> { /// RFM ping/echo modes: pure reflective-memory latency, no EtherCAT. async fn rfm_mode_main(cfg: &Config) -> Result<(), Box> { - println!("rfm: opening PCI-5565 via VFIO..."); - let rfm = el6695_rt::rfm::Rfm::open()?; + let rfm = el6695_rt::rfm::Rfm::open_path(cfg.rfm_dev.as_deref())?; println!( - "rfm: BAR2 reflective mem = {} MB, node_id={}", + "rfm: {} BAR reflective mem = {} MB, node_id={}", + cfg.rfm_dev.as_deref().unwrap_or("/dev/rfm2g0(default)"), rfm.mem.len() / (1024 * 1024), rfm.node_id ); let mem_ptr = rfm.mem.as_ptr() as *mut u8; + // RT-rotate this thread onto its own core (both ping and echo spin; + // sharing a core with the peer would add ~0.5ms scheduling latency). + if let Err(e) = rt::promote_current_thread(cfg.core, cfg.prio, true) { + eprintln!("RT promotion failed (continuing): {}", e); + } + + // Clear both mailboxes so a stale REQ/REP from a previous run cannot + // poison the first cycle's RTT sample. + unsafe { + let zero = [0u8; 32]; + el6695_rt::rfm::dev_write(mem_ptr.add(el6695_rt::rfm::MB_REQ_OFF), &zero); + el6695_rt::rfm::dev_write(mem_ptr.add(el6695_rt::rfm::MB_REP_OFF), &zero); + el6695_rt::rfm::barrier(); + } + let mut rtt = OnlineStats::new("rfm_rtt", 100, 0, 5_000_000); let mut period = OnlineStats::new("period", 100, 0, 10_000_000); let cycle_ns = cfg.cycle_us * 1000; @@ -1507,6 +1528,7 @@ async fn rfm_mode_main(cfg: &Config) -> Result<(), Box> { let start = Instant::now(); let mut last_rep_seq: u64 = 0; let mut timeouts: u64 = 0; + let mut dseq_sum: u64 = 0; let mut last_report = Instant::now(); let mut last_cyc: u64 = 0; @@ -1522,7 +1544,8 @@ async fn rfm_mode_main(cfg: &Config) -> Result<(), Box> { unsafe { let mb = el6695_rt::rfm::Mailbox::at(mem_ptr, el6695_rt::rfm::MB_REQ_OFF); let req_seq = seq.wrapping_add(1); - std::ptr::write_bytes(&mut mb.payload as *mut u8, (seq & 0xFF) as u8, 64); + let pattern = [(seq & 0xFF) as u8; 64]; + el6695_rt::rfm::dev_write(mb.payload.as_mut_ptr(), &pattern); mb.len = 64; mb.ts_ns = ts; el6695_rt::rfm::barrier(); @@ -1538,6 +1561,7 @@ async fn rfm_mode_main(cfg: &Config) -> Result<(), Box> { let rep = el6695_rt::rfm::Mailbox::at(mem_ptr, el6695_rt::rfm::MB_REP_OFF); el6695_rt::rfm::barrier(); if rep.magic == el6695_rt::rfm::MB_MAGIC && rep.seq != last_rep_seq { + dseq_sum += seq.wrapping_add(1).saturating_sub(rep.seq); last_rep_seq = rep.seq; let rtt_ns = el6695_rt::rfm::now_ns().saturating_sub(rep.ts_ns); rtt.push(rtt_ns as i64); @@ -1588,18 +1612,20 @@ async fn rfm_mode_main(cfg: &Config) -> Result<(), Box> { let cur_ts = req.ts_ns; let cur_len = req.len; let mut buf = [0u8; el6695_rt::rfm::MB_MAX_PAYLOAD]; - buf[..n].copy_from_slice(&req.payload[..n]); + el6695_rt::rfm::dev_read(&mut buf[..n], req.payload.as_ptr()); let rep = el6695_rt::rfm::Mailbox::at(mem_ptr, el6695_rt::rfm::MB_REP_OFF); - rep.payload[..n].copy_from_slice(&buf[..n]); + el6695_rt::rfm::dev_write(rep.payload.as_mut_ptr(), &buf[..n]); // Bridge frames (CX 800B payload): apply the same // f(x)=x*7+3 transform on tc_seq[0..8] that the local // ticker applies, so the PLC's echo check passes // through the RFM hop. Small ping frames pass through // verbatim. if (cur_len as usize) >= 280 { - let seq_val = u64::from_le_bytes(rep.payload[..8].try_into().unwrap()); + let mut seq_bytes = [0u8; 8]; + el6695_rt::rfm::dev_read(&mut seq_bytes, rep.payload.as_ptr()); + let seq_val = u64::from_le_bytes(seq_bytes); let processed = seq_val.wrapping_mul(7).wrapping_add(3); - rep.payload[..8].copy_from_slice(&processed.to_le_bytes()); + el6695_rt::rfm::dev_write(rep.payload.as_mut_ptr(), &processed.to_le_bytes()); } rep.len = cur_len; rep.ts_ns = cur_ts; diff --git a/src/rfm.rs b/src/rfm.rs index fcebd40..c3b38b9 100644 --- a/src/rfm.rs +++ b/src/rfm.rs @@ -122,20 +122,31 @@ impl Rfm { /// Open the reflective memory card. Tries the rfm2g kernel char device /// first (/dev/rfm2g0), falls back to VFIO. pub fn open() -> Result> { + Self::open_path(None) + } + + /// Open a specific card by device path (e.g. /dev/rfm2g1), or the default + /// first card when `path` is None. + pub fn open_path(path: Option<&str>) -> Result> { + if let Some(p) = path { + if std::path::Path::new(p).exists() { + return Self::open_chardev(p); + } + return Err(format!("device {} not found", p).into()); + } if std::path::Path::new("/dev/rfm2g0").exists() { - return Self::open_chardev(); + return Self::open_chardev("/dev/rfm2g0"); } Self::open_vfio() } - /// Open via the rfm2g kernel driver (/dev/rfm2g0). + /// Open via the rfm2g kernel driver char device. /// /// The driver's mmap maps the reflective memory at pgoff == byte offset, /// and BAR0 registers when the offset is ORed with RFM2G_BAR0_MMAP_OFFSET /// (== 2). See rfm2g-port driver/driver_entry.c rfm2g_mmap(). - fn open_chardev() -> Result> { - const RFM2G_BAR0_MMAP_OFFSET: usize = 2; - let f = OpenOptions::new().read(true).write(true).open("/dev/rfm2g0")?; + fn open_chardev(path: &str) -> Result> { + let f = OpenOptions::new().read(true).write(true).open(path)?; let fd = f.as_raw_fd(); // Map first 8MB of reflective memory (mailboxes live at offset 0). let map_len = 8 * 1024 * 1024usize; @@ -155,30 +166,14 @@ impl Rfm { }); } let mem = unsafe { core::slice::from_raw_parts_mut(p as *mut u8, map_len) }; - - // BAR0 registers for node id (best effort). - let regs: Option<&'static mut [u8]> = unsafe { - let r = libc::mmap( - core::ptr::null_mut(), - 4096, - libc::PROT_READ | libc::PROT_WRITE, - libc::MAP_SHARED, - fd, - RFM2G_BAR0_MMAP_OFFSET as i64, - ); - if r == libc::MAP_FAILED { - None - } else { - Some(core::slice::from_raw_parts_mut(r as *mut u8, 4096)) - } - }; - let node_id = regs.as_ref().map(|r| r[0x34]).unwrap_or(0xFF); - + // BAR0 register access via this driver requires an ioctl-selected + // mapping (RFM2G_BAR0_MMAP_OFFSET is an ioctl action, not a pgoff); + // skip it — node id is cosmetic. Ok(Self { device_fd: f, - regs, + regs: None, mem, - node_id, + node_id: 0xFF, }) } @@ -330,3 +325,40 @@ pub fn now_ns() -> u64 { unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) }; ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 } + +// ---- device-memory-safe bulk access ---- +// +// The reflective memory is mapped non-cached (pgprot_noncached / +// DEVICE_nGnRnE on ARM64). memcpy/memset on such mappings use SIMD or +// multi-register instructions that fault on ARM64 device memory (SIGBUS). +// Use aligned volatile word/byte loops instead. + +#[inline] +pub unsafe fn dev_write(dst: *mut u8, src: &[u8]) { + let n = src.len(); + let mut i = 0usize; + while i + 8 <= n && (dst.add(i) as usize) % 8 == 0 { + let v = u64::from_le_bytes(src[i..i + 8].try_into().unwrap()); + (dst.add(i) as *mut u64).write_volatile(v); + i += 8; + } + while i < n { + dst.add(i).write_volatile(src[i]); + i += 1; + } +} + +#[inline] +pub unsafe fn dev_read(dst: &mut [u8], src: *const u8) { + let n = dst.len(); + let mut i = 0usize; + while i + 8 <= n && (src.add(i) as usize) % 8 == 0 { + let v = (src.add(i) as *const u64).read_volatile(); + dst[i..i + 8].copy_from_slice(&v.to_le_bytes()); + i += 8; + } + while i < n { + dst[i] = src.add(i).read_volatile(); + i += 1; + } +}