diff --git a/mmdet/visualization/local_visualizer.py b/mmdet/visualization/local_visualizer.py
index cc6521c56eb..88fb95825ef 100644
--- a/mmdet/visualization/local_visualizer.py
+++ b/mmdet/visualization/local_visualizer.py
@@ -278,6 +278,10 @@ def _draw_panoptic_seg(self, image: np.ndarray,
 
         self.set_image(image)
 
+        # if segms is empty, return the image
+        if len(segms) == 0:
+            return self.get_image()
+
         # draw segm
         polygons = []
         for i, mask in enumerate(segms):