refactor: use VirtioInterrupt in VirtIO devices

VirtIO devices assume they're operating under an MMIO transport and as a
consequence they use IrqTrigger as interrupts. Switch that to using
VirtioInterrupt for all VirtIO device objects. Only assume a
VirtioInterrupt is an IrqTrigger in MMIO specific code.

Signed-off-by: Babis Chalios <bchalios@amazon.es>

Author: bchalios

src/vmm/src/device_manager/mmio.rs

@@ -505,7 +505,7 @@ impl MMIODeviceManager {
                             .unwrap();
                         if vsock.is_activated() {
                             info!("kick vsock {id}.");
-                            vsock.signal_used_queue().unwrap();
+                            vsock.signal_used_queue(0).unwrap();
                         }
                     }
                     TYPE_RNG => {
@@ -525,6 +525,7 @@ impl MMIODeviceManager {
 #[cfg(test)]
 mod tests {
 
+    use std::ops::Deref;
     use std::sync::Arc;
 
     use vmm_sys_util::eventfd::EventFd;
@@ -534,6 +535,7 @@ mod tests {
     use crate::devices::virtio::ActivateError;
     use crate::devices::virtio::device::VirtioDevice;
     use crate::devices::virtio::queue::Queue;
+    use crate::devices::virtio::transport::VirtioInterrupt;
     use crate::devices::virtio::transport::mmio::IrqTrigger;
     use crate::test_utils::multi_region_mem_raw;
     use crate::vstate::kvm::Kvm;
@@ -620,10 +622,8 @@ mod tests {
             &self.queue_evts
         }
 
-        fn interrupt_trigger(&self) -> &IrqTrigger {
-            self.interrupt_trigger
-                .as_ref()
-                .expect("Device is not activated")
+        fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
+            self.interrupt_trigger.as_ref().unwrap().deref()
         }
 
         fn ack_features_by_page(&mut self, page: u32, value: u32) {
@@ -644,7 +644,7 @@ mod tests {
         fn activate(
             &mut self,
             _: GuestMemoryMmap,
-            _: Arc<IrqTrigger>,
+            _: Arc<dyn VirtioInterrupt>,
         ) -> Result<(), ActivateError> {
             Ok(())
         }

src/vmm/src/devices/virtio/balloon/device.rs

@@ -25,8 +25,9 @@ use super::{
     VIRTIO_BALLOON_S_SWAP_OUT,
 };
 use crate::devices::virtio::balloon::BalloonError;
+use crate::devices::virtio::device::ActiveState;
 use crate::devices::virtio::generated::virtio_config::VIRTIO_F_VERSION_1;
-use crate::devices::virtio::transport::mmio::{IrqTrigger, IrqType};
+use crate::devices::virtio::transport::{VirtioInterrupt, VirtioInterruptType};
 use crate::logger::IncMetric;
 use crate::utils::u64_to_usize;
 use crate::vstate::memory::{Address, ByteValued, Bytes, GuestAddress, GuestMemoryMmap};
@@ -259,7 +260,7 @@ impl Balloon {
 
     pub(crate) fn process_inflate(&mut self) -> Result<(), BalloonError> {
         // This is safe since we checked in the event handler that the device is activated.
-        let (mem, _) = self.device_state.active_state().unwrap();
+        let mem = &self.device_state.active_state().unwrap().mem;
         METRICS.inflate_count.inc();
 
         let queue = &mut self.queues[INFLATE_INDEX];
@@ -340,7 +341,7 @@ impl Balloon {
         }
 
         if needs_interrupt {
-            self.signal_used_queue()?;
+            self.signal_used_queue(INFLATE_INDEX)?;
         }
 
         Ok(())
@@ -358,15 +359,15 @@ impl Balloon {
         }
 
         if needs_interrupt {
-            self.signal_used_queue()
+            self.signal_used_queue(DEFLATE_INDEX)
         } else {
             Ok(())
         }
     }
 
     pub(crate) fn process_stats_queue(&mut self) -> Result<(), BalloonError> {
         // This is safe since we checked in the event handler that the device is activated.
-        let (mem, _) = self.device_state.active_state().unwrap();
+        let mem = &self.device_state.active_state().unwrap().mem;
         METRICS.stats_updates_count.inc();
 
         while let Some(head) = self.queues[STATS_INDEX].pop() {
@@ -402,9 +403,12 @@ impl Balloon {
         Ok(())
     }
 
-    pub(crate) fn signal_used_queue(&self) -> Result<(), BalloonError> {
+    pub(crate) fn signal_used_queue(&self, qidx: usize) -> Result<(), BalloonError> {
         self.interrupt_trigger()
-            .trigger_irq(IrqType::Vring)
+            .trigger(VirtioInterruptType::Queue(
+                qidx.try_into()
+                    .unwrap_or_else(|_| panic!("balloon: invalid queue id: {qidx}")),
+            ))
             .map_err(|err| {
                 METRICS.event_fails.inc();
                 BalloonError::InterruptError(err)
@@ -429,7 +433,7 @@ impl Balloon {
             self.queues[STATS_INDEX]
                 .add_used(index, 0)
                 .map_err(BalloonError::Queue)?;
-            self.signal_used_queue()
+            self.signal_used_queue(STATS_INDEX)
         } else {
             error!("Failed to update balloon stats, missing descriptor.");
             Ok(())
@@ -441,7 +445,7 @@ impl Balloon {
         if self.is_activated() {
             self.config_space.num_pages = mib_to_pages(amount_mib)?;
             self.interrupt_trigger()
-                .trigger_irq(IrqType::Config)
+                .trigger(VirtioInterruptType::Config)
                 .map_err(BalloonError::InterruptError)
         } else {
             Err(BalloonError::DeviceNotActive)
@@ -552,11 +556,11 @@ impl VirtioDevice for Balloon {
         &self.queue_evts
     }
 
-    fn interrupt_trigger(&self) -> &IrqTrigger {
+    fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
         self.device_state
             .active_state()
             .expect("Device is not activated")
-            .1
+            .interrupt
             .deref()
     }
 
@@ -587,14 +591,14 @@ impl VirtioDevice for Balloon {
     fn activate(
         &mut self,
         mem: GuestMemoryMmap,
-        interrupt: Arc<IrqTrigger>,
+        interrupt: Arc<dyn VirtioInterrupt>,
     ) -> Result<(), ActivateError> {
         for q in self.queues.iter_mut() {
             q.initialize(&mem)
                 .map_err(ActivateError::QueueMemoryError)?;
         }
 
-        self.device_state = DeviceState::Activated((mem, interrupt));
+        self.device_state = DeviceState::Activated(ActiveState { mem, interrupt });
         if self.activate_evt.write(1).is_err() {
             METRICS.activate_fails.inc();
             self.device_state = DeviceState::Inactive;
@@ -1059,7 +1063,9 @@ pub(crate) mod tests {
                 assert!(balloon.stats_desc_index.is_some());
                 balloon.process_stats_timer_event().unwrap();
                 assert!(balloon.stats_desc_index.is_none());
-                assert!(balloon.interrupt_trigger().has_pending_irq(IrqType::Vring));
+                assert!(balloon.interrupt_trigger().has_pending_interrupt(
+                    VirtioInterruptType::Queue(STATS_INDEX.try_into().unwrap())
+                ));
             });
         }
     }
@@ -1114,8 +1120,10 @@ pub(crate) mod tests {
     fn test_num_pages() {
         let mut balloon = Balloon::new(0, true, 0, false).unwrap();
         // Switch the state to active.
-        balloon.device_state =
-            DeviceState::Activated((single_region_mem(0x1), default_interrupt()));
+        balloon.device_state = DeviceState::Activated(ActiveState {
+            mem: single_region_mem(0x1),
+            interrupt: default_interrupt(),
+        });
 
         assert_eq!(balloon.num_pages(), 0);
         assert_eq!(balloon.actual_pages(), 0);

src/vmm/src/devices/virtio/balloon/persist.rs

@@ -12,10 +12,10 @@ use timerfd::{SetTimeFlags, TimerState};
 use super::*;
 use crate::devices::virtio::TYPE_BALLOON;
 use crate::devices::virtio::balloon::device::{BalloonStats, ConfigSpace};
-use crate::devices::virtio::device::DeviceState;
+use crate::devices::virtio::device::{ActiveState, DeviceState};
 use crate::devices::virtio::persist::VirtioDeviceState;
 use crate::devices::virtio::queue::FIRECRACKER_MAX_QUEUE_SIZE;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
+use crate::devices::virtio::transport::VirtioInterrupt;
 use crate::snapshot::Persist;
 use crate::vstate::memory::GuestMemoryMmap;
 
@@ -96,7 +96,7 @@ pub struct BalloonConstructorArgs {
     /// Pointer to guest memory.
     pub mem: GuestMemoryMmap,
     /// Interrupt used from the device.
-    pub interrupt: Arc<IrqTrigger>,
+    pub interrupt: Arc<dyn VirtioInterrupt>,
     pub restored_from_file: bool,
 }
 
@@ -155,8 +155,10 @@ impl Persist<'_> for Balloon {
         };
 
         if state.virtio_state.activated {
-            balloon.device_state =
-                DeviceState::Activated((constructor_args.mem, constructor_args.interrupt));
+            balloon.device_state = DeviceState::Activated(ActiveState {
+                mem: constructor_args.mem,
+                interrupt: constructor_args.interrupt,
+            });
 
             if balloon.stats_enabled() {
                 // Restore the stats descriptor.

src/vmm/src/devices/virtio/balloon/test_utils.rs

@@ -3,14 +3,16 @@
 
 #![doc(hidden)]
 
+#[cfg(test)]
+use crate::devices::virtio::device::VirtioDevice;
 use crate::devices::virtio::test_utils::VirtQueue;
 #[cfg(test)]
 use crate::devices::virtio::{balloon::BALLOON_NUM_QUEUES, balloon::Balloon};
 
 #[cfg(test)]
 pub fn invoke_handler_for_queue_event(b: &mut Balloon, queue_index: usize) {
     use crate::devices::virtio::balloon::{DEFLATE_INDEX, INFLATE_INDEX, STATS_INDEX};
-    use crate::devices::virtio::transport::mmio::IrqType;
+    use crate::devices::virtio::transport::VirtioInterruptType;
 
     assert!(queue_index < BALLOON_NUM_QUEUES);
     // Trigger the queue event.
@@ -23,8 +25,11 @@ pub fn invoke_handler_for_queue_event(b: &mut Balloon, queue_index: usize) {
         _ => unreachable!(),
     };
     // Validate the queue operation finished successfully.
-    let (_, interrupt) = b.device_state.active_state().unwrap();
-    assert!(interrupt.has_pending_irq(IrqType::Vring));
+    let interrupt = b.interrupt_trigger();
+    assert!(
+        interrupt
+            .has_pending_interrupt(VirtioInterruptType::Queue(queue_index.try_into().unwrap()))
+    );
 }
 
 pub fn set_request(queue: &VirtQueue, idx: u16, addr: u64, len: u32, flags: u16) {

src/vmm/src/devices/virtio/block/device.rs

@@ -12,7 +12,7 @@ use super::vhost_user::device::{VhostUserBlock, VhostUserBlockConfig};
 use super::virtio::device::{VirtioBlock, VirtioBlockConfig};
 use crate::devices::virtio::device::VirtioDevice;
 use crate::devices::virtio::queue::Queue;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
+use crate::devices::virtio::transport::VirtioInterrupt;
 use crate::devices::virtio::{ActivateError, TYPE_BLOCK};
 use crate::rate_limiter::BucketUpdate;
 use crate::snapshot::Persist;
@@ -176,7 +176,7 @@ impl VirtioDevice for Block {
         }
     }
 
-    fn interrupt_trigger(&self) -> &IrqTrigger {
+    fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
         match self {
             Self::Virtio(b) => b.interrupt_trigger(),
             Self::VhostUser(b) => b.interrupt_trigger(),
@@ -200,7 +200,7 @@ impl VirtioDevice for Block {
     fn activate(
         &mut self,
         mem: GuestMemoryMmap,
-        interrupt: Arc<IrqTrigger>,
+        interrupt: Arc<dyn VirtioInterrupt>,
     ) -> Result<(), ActivateError> {
         match self {
             Self::Virtio(b) => b.activate(mem, interrupt),

src/vmm/src/devices/virtio/block/persist.rs

@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
 
 use super::vhost_user::persist::VhostUserBlockState;
 use super::virtio::persist::VirtioBlockState;
-use crate::devices::virtio::transport::mmio::IrqTrigger;
+use crate::devices::virtio::transport::VirtioInterrupt;
 use crate::vstate::memory::GuestMemoryMmap;
 
 /// Block device state.
@@ -21,5 +21,5 @@ pub enum BlockState {
 #[derive(Debug)]
 pub struct BlockConstructorArgs {
     pub mem: GuestMemoryMmap,
-    pub interrupt: Arc<IrqTrigger>,
+    pub interrupt: Arc<dyn VirtioInterrupt>,
 }

src/vmm/src/devices/virtio/block/vhost_user/device.rs

@@ -15,12 +15,12 @@ use vmm_sys_util::eventfd::EventFd;
 
 use super::{NUM_QUEUES, QUEUE_SIZE, VhostUserBlockError};
 use crate::devices::virtio::block::CacheType;
-use crate::devices::virtio::device::{DeviceState, VirtioDevice};
+use crate::devices::virtio::device::{ActiveState, DeviceState, VirtioDevice};
 use crate::devices::virtio::generated::virtio_blk::{VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_RO};
 use crate::devices::virtio::generated::virtio_config::VIRTIO_F_VERSION_1;
 use crate::devices::virtio::generated::virtio_ring::VIRTIO_RING_F_EVENT_IDX;
 use crate::devices::virtio::queue::Queue;
-use crate::devices::virtio::transport::mmio::{IrqTrigger, IrqType};
+use crate::devices::virtio::transport::{VirtioInterrupt, VirtioInterruptType};
 use crate::devices::virtio::vhost_user::{VhostUserHandleBackend, VhostUserHandleImpl};
 use crate::devices::virtio::vhost_user_metrics::{
     VhostUserDeviceMetrics, VhostUserMetricsPerDevice,
@@ -254,10 +254,12 @@ impl<T: VhostUserHandleBackend> VhostUserBlockImpl<T> {
 
     pub fn config_update(&mut self) -> Result<(), VhostUserBlockError> {
         let start_time = get_time_us(ClockType::Monotonic);
-        let (_, interrupt) = self
+        let interrupt = self
             .device_state
             .active_state()
-            .expect("Device is not initialized");
+            .expect("Device is not initialized")
+            .interrupt
+            .clone();
 
         // This buffer is used for config size check in vhost crate.
         let buffer = [0u8; BLOCK_CONFIG_SPACE_SIZE as usize];
@@ -273,8 +275,8 @@ impl<T: VhostUserHandleBackend> VhostUserBlockImpl<T> {
             .map_err(VhostUserBlockError::Vhost)?;
         self.config_space = new_config_space;
         interrupt
-            .trigger_irq(IrqType::Config)
-            .map_err(VhostUserBlockError::IrqTrigger)?;
+            .trigger(VirtioInterruptType::Config)
+            .map_err(VhostUserBlockError::Interrupt)?;
 
         let delta_us = get_time_us(ClockType::Monotonic) - start_time;
         self.metrics.config_change_time_us.store(delta_us);
@@ -312,11 +314,11 @@ impl<T: VhostUserHandleBackend + Send + 'static> VirtioDevice for VhostUserBlock
         &self.queue_evts
     }
 
-    fn interrupt_trigger(&self) -> &IrqTrigger {
+    fn interrupt_trigger(&self) -> &dyn VirtioInterrupt {
         self.device_state
             .active_state()
             .expect("Device is not initialized")
-            .1
+            .interrupt
             .deref()
     }
 
@@ -339,7 +341,7 @@ impl<T: VhostUserHandleBackend + Send + 'static> VirtioDevice for VhostUserBlock
     fn activate(
         &mut self,
         mem: GuestMemoryMmap,
-        interrupt: Arc<IrqTrigger>,
+        interrupt: Arc<dyn VirtioInterrupt>,
     ) -> Result<(), ActivateError> {
         for q in self.queues.iter_mut() {
             q.initialize(&mem)
@@ -355,14 +357,14 @@ impl<T: VhostUserHandleBackend + Send + 'static> VirtioDevice for VhostUserBlock
                 self.vu_handle.setup_backend(
                     &mem,
                     &[(0, &self.queues[0], &self.queue_evts[0])],
-                    &interrupt,
+                    interrupt.clone(),
                 )
             })
             .map_err(|err| {
                 self.metrics.activate_fails.inc();
                 ActivateError::VhostUser(err)
             })?;
-        self.device_state = DeviceState::Activated((mem, interrupt));
+        self.device_state = DeviceState::Activated(ActiveState { mem, interrupt });
         let delta_us = get_time_us(ClockType::Monotonic) - start_time;
         self.metrics.activate_time_us.store(delta_us);
         Ok(())
@@ -662,7 +664,10 @@ mod tests {
         assert_eq!(vhost_block.config_space, vec![0x69, 0x69, 0x69]);
 
         // Testing [`config_update`]
-        vhost_block.device_state = DeviceState::Activated((default_mem(), default_interrupt()));
+        vhost_block.device_state = DeviceState::Activated(ActiveState {
+            mem: default_mem(),
+            interrupt: default_interrupt(),
+        });
         vhost_block.config_space = vec![];
         vhost_block.config_update().unwrap();
         assert_eq!(vhost_block.config_space, vec![0x69, 0x69, 0x69]);

src/vmm/src/devices/virtio/block/vhost_user/mod.rs

@@ -28,5 +28,5 @@ pub enum VhostUserBlockError {
     /// Error opening eventfd: {0}
     EventFd(std::io::Error),
     /// Error creating irqfd: {0}
-    IrqTrigger(std::io::Error),
+    Interrupt(std::io::Error),
 }