ethercat-linux/src/rfm.rs

//! GE/Abaco PCI-5565 (RFM2G) reflective memory access via VFIO.
//!
//! The card is bound to vfio-pci at boot (`/sbin/rfm2g-vfio-bind.sh`).
//! This module opens the VFIO container + group, obtains the device fd,
//! and mmaps BAR0 (registers) and BAR2 (64MB reflective memory).
//! Works in both SMMU (IOMMU groups) and noiommu fallback modes.

use std::fs::{File, OpenOptions};
use std::io;
use std::os::unix::io::AsRawFd;

pub const VENDOR: u16 = 0x114a;
pub const DEVICE: u16 = 0x5565;

// ---- vfio ioctls (include/uapi/linux/vfio.h) ----
const VFIO_TYPE: u8 = b';';
const VFIO_BASE: u8 = 100;

const VFIO_GET_API_VERSION: u32 = ior(VFIO_TYPE, VFIO_BASE + 0, 0);
const VFIO_GROUP_GET_DEVICE_FD: u32 = iow(VFIO_TYPE, VFIO_BASE + 6, 4);
const VFIO_DEVICE_GET_INFO: u32 = ior(VFIO_TYPE, VFIO_BASE + 7, 16);
const VFIO_DEVICE_GET_REGION_INFO: u32 = iow(VFIO_TYPE, VFIO_BASE + 8, 32);

const VFIO_API_VERSION: i32 = 0;

#[repr(C)]
#[derive(Default)]
struct VfioDeviceInfo {
    argsz: u32,
    flags: u32,
    num_regions: u32,
    num_irqs: u32,
}

#[repr(C)]
#[derive(Default, Clone, Copy)]
pub struct VfioRegionInfo {
    pub argsz: u32,
    pub flags: u32,
    pub index: u32,
    pub cap_offset: u32,
    pub size: u64,
    pub offset: u64,
}

const fn ioc(dir: u32, ty: u8, nr: u8, size: u32) -> u32 {
    (dir << 30) | ((ty as u32) << 8) | (nr as u32) | (size << 16)
}
const fn ior(ty: u8, nr: u8, size: u32) -> u32 {
    ioc(2, ty, nr, size)
}
const fn iow(ty: u8, nr: u8, size: u32) -> u32 {
    ioc(1, ty, nr, size)
}

fn ioctl(fd: i32, req: u32, arg: usize) -> io::Result<i32> {
    // libc::ioctl takes `Ioctl` (= c_ulong on gnu, i32 on musl) as request.
    #[allow(clippy::useless_conversion)]
    let r = unsafe { libc::ioctl(fd, req as _, arg) };
    if r < 0 {
        Err(io::Error::last_os_error())
    } else {
        Ok(r)
    }
}

pub struct Rfm {
    #[allow(dead_code)]
    device_fd: File,
    /// BAR0 (registers) mapping, if present.
    pub regs: Option<&'static mut [u8]>,
    /// BAR2 reflective memory mapping.
    pub mem: &'static mut [u8],
    /// Node ID read from BAR0 (byte at offset 0x34 on PCI-5565 family).
    pub node_id: u8,
}

fn find_bdf() -> io::Result<String> {
    for ent in std::fs::read_dir("/sys/bus/pci/devices")? {
        let ent = ent?;
        let p = ent.path();
        let rd = |n: &str| -> Option<u32> {
            std::fs::read_to_string(p.join(n))
                .ok()
                .and_then(|s| u32::from_str_radix(s.trim().trim_start_matches("0x"), 16).ok())
        };
        if rd("vendor") == Some(VENDOR as u32) && rd("device") == Some(DEVICE as u32) {
            return Ok(ent.file_name().to_string_lossy().into_owned());
        }
    }
    Err(io::Error::new(
        io::ErrorKind::NotFound,
        format!("PCI-5565 ({:#06x}:{:#06x}) not present", VENDOR, DEVICE),
    ))
}

fn iommu_group_of(bdf: &str) -> io::Result<String> {
    let link = format!("/sys/bus/pci/devices/{}/iommu_group", bdf);
    let t = std::fs::read_link(&link)
        .map_err(|e| io::Error::new(e.kind(), format!("readlink {}: {}", link, e)))?;
    Ok(t.file_name().unwrap().to_string_lossy().into_owned())
}

unsafe fn mmap_bar(fd: i32, index: u32, info: &VfioRegionInfo) -> io::Result<&'static mut [u8]> {
    let map_off = (index as u64) << 40; // VFIO region mmap offset convention
    let len = info.size as usize;
    let p = libc::mmap(
        core::ptr::null_mut(),
        len,
        libc::PROT_READ | libc::PROT_WRITE,
        libc::MAP_SHARED,
        fd,
        (info.offset + map_off) as i64,
    );
    if p == libc::MAP_FAILED {
        return Err(io::Error::last_os_error());
    }
    Ok(core::slice::from_raw_parts_mut(p as *mut u8, len))
}

impl Rfm {
    /// Open the reflective memory card. Tries the rfm2g kernel char device
    /// first (/dev/rfm2g0), falls back to VFIO.
    pub fn open() -> Result<Self, Box<dyn std::error::Error>> {
        Self::open_path(None)
    }

    /// Open a specific card by device path (e.g. /dev/rfm2g1), or the default
    /// first card when `path` is None.
    pub fn open_path(path: Option<&str>) -> Result<Self, Box<dyn std::error::Error>> {
        if let Some(p) = path {
            if std::path::Path::new(p).exists() {
                return Self::open_chardev(p);
            }
            return Err(format!("device {} not found", p).into());
        }
        if std::path::Path::new("/dev/rfm2g0").exists() {
            return Self::open_chardev("/dev/rfm2g0");
        }
        Self::open_vfio()
    }

    /// Open via the rfm2g kernel driver char device.
    ///
    /// The driver's mmap maps the reflective memory at pgoff == byte offset,
    /// and BAR0 registers when the offset is ORed with RFM2G_BAR0_MMAP_OFFSET
    /// (== 2). See rfm2g-port driver/driver_entry.c rfm2g_mmap().
    fn open_chardev(path: &str) -> Result<Self, Box<dyn std::error::Error>> {
        let f = OpenOptions::new().read(true).write(true).open(path)?;
        let fd = f.as_raw_fd();
        // Map first 8MB of reflective memory (mailboxes live at offset 0).
        let map_len = 8 * 1024 * 1024usize;
        let p = unsafe {
            libc::mmap(
                core::ptr::null_mut(),
                map_len,
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_SHARED,
                fd,
                0,
            )
        };
        if p == libc::MAP_FAILED {
            return Err(io::Error::last_os_error()).map_err(|e| {
                format!("rfm2g mmap failed ({}): driver loaded? card present?", e).into()
            });
        }
        let mem = unsafe { core::slice::from_raw_parts_mut(p as *mut u8, map_len) };
        // BAR0 register access via this driver requires an ioctl-selected
        // mapping (RFM2G_BAR0_MMAP_OFFSET is an ioctl action, not a pgoff);
        // skip it — node id is cosmetic.
        Ok(Self {
            device_fd: f,
            regs: None,
            mem,
            node_id: 0xFF,
        })
    }

    /// Open the reflective memory card via VFIO and map BARs.
    fn open_vfio() -> Result<Self, Box<dyn std::error::Error>> {
        let bdf = find_bdf()?;
        let group = iommu_group_of(&bdf).unwrap_or_else(|_| {
            // noiommu fallback: group devices are named noiommu-N; find ours by
            // matching the device link inside each noiommu group.
            let mut found = String::new();
            if let Ok(dir) = std::fs::read_dir("/sys/class/vfio") {
                for e in dir.flatten() {
                    let g = e.file_name().to_string_lossy().into_owned();
                    if g.starts_with("noiommu-") {
                        let dev_link = format!("/sys/class/vfio/{}/devices/{}", g, bdf);
                        if std::path::Path::new(&dev_link).exists() {
                            found = g;
                            break;
                        }
                    }
                }
            }
            found
        });
        if group.is_empty() {
            return Err(format!("no IOMMU/noiommu group for {}", bdf).into());
        }

        let container = OpenOptions::new().read(true).write(true).open("/dev/vfio/vfio")?;
        let group_path = format!("/dev/vfio/{}", group);
        let group_fd = OpenOptions::new().read(true).write(true).open(&group_path)?;

        let ver = ioctl(container.as_raw_fd(), VFIO_GET_API_VERSION, 0)?;
        if ver != VFIO_API_VERSION {
            return Err(format!("vfio api version {} != {}", ver, VFIO_API_VERSION).into());
        }

        // VFIO_GROUP_SET_CONTAINER (no IOMMU ops needed in noiommu mode)
        const VFIO_GROUP_SET_CONTAINER: u32 = iow(VFIO_TYPE, VFIO_BASE + 4, 4);
        let cfd = container.as_raw_fd();
        ioctl(group_fd.as_raw_fd(), VFIO_GROUP_SET_CONTAINER, &cfd as *const i32 as usize)?;

        let mut dev_name = bdf.clone().into_bytes();
        dev_name.push(0);
        let dev_fd = ioctl(
            group_fd.as_raw_fd(),
            VFIO_GROUP_GET_DEVICE_FD,
            dev_name.as_ptr() as usize,
        )?;
        if dev_fd <= 0 {
            return Err("VFIO_GROUP_GET_DEVICE_FD failed".into());
        }
        let device_fd = unsafe { File::from_raw_fd(dev_fd) };

        let mut dinfo = VfioDeviceInfo {
            argsz: core::mem::size_of::<VfioDeviceInfo>() as u32,
            ..Default::default()
        };
        ioctl(
            device_fd.as_raw_fd(),
            VFIO_DEVICE_GET_INFO,
            &mut dinfo as *mut _ as usize,
        )?;

        let mut regs: Option<&'static mut [u8]> = None;
        let mut mem: Option<&'static mut [u8]> = None;
        for idx in 0..dinfo.num_regions {
            let mut rinfo = VfioRegionInfo {
                argsz: core::mem::size_of::<VfioRegionInfo>() as u32,
                index: idx,
                ..Default::default()
            };
            if ioctl(
                device_fd.as_raw_fd(),
                VFIO_DEVICE_GET_REGION_INFO,
                &mut rinfo as *mut _ as usize,
            )
            .is_err()
            {
                continue;
            }
            if rinfo.size == 0 {
                continue;
            }
            // PCI-5565: BAR0 = registers (small), BAR2 = 64MB reflective RAM
            if idx == 0 {
                if let Ok(m) = unsafe { mmap_bar(device_fd.as_raw_fd(), idx, &rinfo) } {
                    regs = Some(m);
                }
            } else if idx == 2 && mem.is_none() {
                mem = Some(unsafe { mmap_bar(device_fd.as_raw_fd(), idx, &rinfo)? });
            }
        }

        let mem = mem.ok_or("BAR2 (reflective memory) not mappable")?;
        let node_id = regs.as_ref().map(|r| r[0x34]).unwrap_or(0xFF);

        Ok(Self {
            device_fd,
            regs,
            mem,
            node_id,
        })
    }
}

use std::os::unix::io::FromRawFd;

// ---- mailbox protocol over reflective memory ----
//
// Two fixed mailboxes inside BAR2. Writer fills header fields and payload,
// then bumps `seq` LAST (release). Reader polls `seq`, then reads (acquire).
// x86 and aarch64 both see coherent views of the card's onboard RAM.

pub const MB_REQ_OFF: usize = 0x0000;
pub const MB_REP_OFF: usize = 0x1000;
pub const MB_MAGIC: u32 = 0x5246_4D32; // "RFM2"
pub const MB_MAX_PAYLOAD: usize = 1024;

#[repr(C)]
pub struct Mailbox {
    pub magic: u32,
    pub len: u32,
    pub seq: u64,
    pub ts_ns: u64,
    pub payload: [u8; MB_MAX_PAYLOAD],
}

impl Mailbox {
    pub unsafe fn at<'a>(mem: *mut u8, off: usize) -> &'a mut Mailbox {
        &mut *(mem.add(off) as *mut Mailbox)
    }
}

#[inline]
pub fn barrier() {
    // Full memory barrier on both x86 and aarch64.
    core::sync::atomic::fence(Ordering::SeqCst);
}

use core::sync::atomic::Ordering;

#[inline]
pub fn now_ns() -> u64 {
    let mut ts = libc::timespec {
        tv_sec: 0,
        tv_nsec: 0,
    };
    unsafe { libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut ts) };
    ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64
}

// ---- device-memory-safe bulk access ----
//
// The reflective memory is mapped non-cached (pgprot_noncached /
// DEVICE_nGnRnE on ARM64). memcpy/memset on such mappings use SIMD or
// multi-register instructions that fault on ARM64 device memory (SIGBUS).
// Use aligned volatile word/byte loops instead.

#[inline]
pub unsafe fn dev_write(dst: *mut u8, src: &[u8]) {
    let n = src.len();
    let mut i = 0usize;
    while i + 8 <= n && (dst.add(i) as usize) % 8 == 0 {
        let v = u64::from_le_bytes(src[i..i + 8].try_into().unwrap());
        (dst.add(i) as *mut u64).write_volatile(v);
        i += 8;
    }
    while i < n {
        dst.add(i).write_volatile(src[i]);
        i += 1;
    }
}

#[inline]
pub unsafe fn dev_read(dst: &mut [u8], src: *const u8) {
    let n = dst.len();
    let mut i = 0usize;
    while i + 8 <= n && (src.add(i) as usize) % 8 == 0 {
        let v = (src.add(i) as *const u64).read_volatile();
        dst[i..i + 8].copy_from_slice(&v.to_le_bytes());
        i += 8;
    }
    while i < n {
        dst[i] = src.add(i).read_volatile();
        i += 1;
    }
}