Output events when widgets gain keyboard focus

Part of https://github.com/emilk/egui/issues/167

CHANGELOG.md

@@ -12,8 +12,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added ⭐
 
 * You can now give focus to any clickable widget with tab/shift-tab.
-  * Use space or enter to click it.
+  * Use space or enter to click the selected widget.
   * Use arrow keys to adjust sliders and `DragValue`s.
+* egui will now output events when widgets gain keyboard focus.
+  * This can be hooked up to a screen reader to aid the visually impaired
 
 ### Fixed 🐛
 

egui/src/containers/collapsing_header.rs

@@ -207,6 +207,10 @@ impl CollapsingHeader {
         let (_, rect) = ui.allocate_space(desired_size);
 
         let header_response = ui.interact(rect, id, Sense::click());
+        if header_response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::CollapsingHeader, &galley.text);
+        }
         let text_pos = pos2(
             text_pos.x,
             header_response.rect.center().y - galley.size.y / 2.0,

egui/src/containers/combo_box.rs

@@ -27,6 +27,10 @@ pub fn combo_box_with_label(
 
     ui.horizontal(|ui| {
         let mut response = combo_box(ui, button_id, selected, menu_contents);
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::ComboBox, label.text());
+        }
         response |= ui.add(label);
         response
     })

egui/src/data/output.rs

@@ -20,6 +20,9 @@ pub struct Output {
     /// As an egui user: don't set this value directly.
     /// Call `Context::request_repaint()` instead and it will do so for you.
     pub needs_repaint: bool,
+
+    /// Events that may be useful to e.g. a screen reader.
+    pub events: Vec<OutputEvent>,
 }
 
 /// A mouse cursor icon.
@@ -45,3 +48,42 @@ impl Default for CursorIcon {
         Self::Default
     }
 }
+
+/// Things that happened during this frame that the integration may be interested in.
+///
+/// In particular, these events may be useful for accessability, i.e. for screen readers.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum OutputEvent {
+    /// A widget gained keyboard focus (by tab key).
+    ///
+    /// An integration can for instance read the newly selected widget out loud for the visually impaired.
+    //
+    // TODO: we should output state too, e.g. if a checkbox is selected, or current slider value.
+    Focused(WidgetType, String),
+}
+
+/// The different types of built-in widgets in egui
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum WidgetType {
+    Label,
+    Hyperlink,
+    TextEdit,
+    Button,
+    Checkbox,
+    RadioButton,
+    SelectableLabel,
+    ComboBox,
+    Slider,
+    DragValue,
+    ColorButton,
+    ImageButton,
+    CollapsingHeader,
+}
+
+impl Output {
+    /// Inform the backend integration that a widget gained focus
+    pub fn push_gained_focus_event(&mut self, widget_type: WidgetType, text: impl Into<String>) {
+        self.events
+            .push(OutputEvent::Focused(widget_type, text.into()));
+    }
+}

egui/src/widgets/button.rs

@@ -119,6 +119,10 @@ impl Button {
         }
 
         let (rect, response) = ui.allocate_at_least(desired_size, sense);
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::TextEdit, &galley.text);
+        }
 
         if ui.clip_rect().intersects(rect) {
             let visuals = ui.style().interact(&response);
@@ -228,6 +232,10 @@ impl<'a> Widget for Checkbox<'a> {
         desired_size = desired_size.at_least(spacing.interact_size);
         desired_size.y = desired_size.y.max(icon_width);
         let (rect, mut response) = ui.allocate_exact_size(desired_size, Sense::click());
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::Checkbox, &galley.text);
+        }
 
         if response.clicked() {
             *checked = !*checked;
@@ -338,6 +346,10 @@ impl Widget for RadioButton {
         desired_size = desired_size.at_least(ui.spacing().interact_size);
         desired_size.y = desired_size.y.max(icon_width);
         let (rect, response) = ui.allocate_exact_size(desired_size, Sense::click());
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::RadioButton, &galley.text);
+        }
 
         let text_cursor = pos2(
             rect.min.x + button_padding.x + icon_width + icon_spacing,
@@ -442,6 +454,10 @@ impl Widget for ImageButton {
         let button_padding = ui.spacing().button_padding;
         let size = image.size() + 2.0 * button_padding;
         let (rect, response) = ui.allocate_exact_size(size, sense);
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::ImageButton, "");
+        }
 
         if ui.clip_rect().intersects(rect) {
             let visuals = ui.style().interact(&response);

egui/src/widgets/color_picker.rs

@@ -66,6 +66,10 @@ fn show_hsva(ui: &mut Ui, color: Hsva, desired_size: Vec2) -> Response {
 fn color_button(ui: &mut Ui, color: Color32) -> Response {
     let size = ui.spacing().interact_size;
     let (rect, response) = ui.allocate_exact_size(size, Sense::click());
+    if response.gained_kb_focus() {
+        ui.output()
+            .push_gained_focus_event(WidgetType::ColorButton, "");
+    }
     let visuals = ui.style().interact(&response);
     let rect = rect.expand(visuals.expansion);
 

egui/src/widgets/drag_value.rs

@@ -298,6 +298,11 @@ impl<'a> Widget for DragValue<'a> {
             response
         };
 
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::DragValue, "");
+        }
+
         #[allow(clippy::float_cmp)]
         {
             response.changed = get(&mut get_set_value) != value;

egui/src/widgets/hyperlink.rs

@@ -53,6 +53,10 @@ impl Widget for Hyperlink {
         let Hyperlink { url, label } = self;
         let galley = label.layout(ui);
         let (rect, response) = ui.allocate_exact_size(galley.size, Sense::click());
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::Hyperlink, &galley.text);
+        }
 
         if response.hovered() {
             ui.ctx().output().cursor_icon = CursorIcon::PointingHand;

egui/src/widgets/selected_label.rs

@@ -55,6 +55,10 @@ impl Widget for SelectableLabel {
         let mut desired_size = total_extra + galley.size;
         desired_size.y = desired_size.y.at_least(ui.spacing().interact_size.y);
         let (rect, response) = ui.allocate_at_least(desired_size, Sense::click());
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::SelectableLabel, &galley.text);
+        }
 
         let text_cursor = ui
             .layout()

egui/src/widgets/slider.rs

@@ -322,6 +322,11 @@ impl<'a> Slider<'a> {
             self.set_value(new_value);
         }
 
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::Slider, &self.text);
+        }
+
         if response.has_kb_focus() {
             let kb_step = ui.input().num_presses(Key::ArrowRight) as f32
                 - ui.input().num_presses(Key::ArrowLeft) as f32;

egui/src/widgets/text_edit.rs

@@ -321,6 +321,10 @@ impl<'t> TextEdit<'t> {
             Sense::hover()
         };
         let mut response = ui.interact(rect, id, sense);
+        if response.gained_kb_focus() {
+            ui.output()
+                .push_gained_focus_event(WidgetType::TextEdit, &*text);
+        }
 
         if enabled {
             if let Some(pointer_pos) = ui.input().pointer.interact_pos() {

egui_demo_lib/src/apps/demo/tests.rs

@@ -93,8 +93,7 @@ impl super::Demo for ManualLayoutTest {
 
 impl super::View for ManualLayoutTest {
     fn ui(&mut self, ui: &mut egui::Ui) {
-        use egui::*;
-        reset_button(ui, self);
+        egui::reset_button(ui, self);
         let Self {
             widget_offset,
             widget_size,
@@ -107,29 +106,30 @@ impl super::View for ManualLayoutTest {
             ui.radio_value(widget_type, WidgetType::Label, "Label");
             ui.radio_value(widget_type, WidgetType::TextEdit, "TextEdit");
         });
-        Grid::new("pos_size").show(ui, |ui| {
+        egui::Grid::new("pos_size").show(ui, |ui| {
             ui.label("Widget position:");
-            ui.add(Slider::f32(&mut widget_offset.x, 0.0..=400.0));
-            ui.add(Slider::f32(&mut widget_offset.y, 0.0..=400.0));
+            ui.add(egui::Slider::f32(&mut widget_offset.x, 0.0..=400.0));
+            ui.add(egui::Slider::f32(&mut widget_offset.y, 0.0..=400.0));
             ui.end_row();
 
             ui.label("Widget size:");
-            ui.add(Slider::f32(&mut widget_size.x, 0.0..=400.0));
-            ui.add(Slider::f32(&mut widget_size.y, 0.0..=400.0));
+            ui.add(egui::Slider::f32(&mut widget_size.x, 0.0..=400.0));
+            ui.add(egui::Slider::f32(&mut widget_size.y, 0.0..=400.0));
             ui.end_row();
         });
-        let widget_rect = Rect::from_min_size(ui.min_rect().min + *widget_offset, *widget_size);
+        let widget_rect =
+            egui::Rect::from_min_size(ui.min_rect().min + *widget_offset, *widget_size);
 
         // Showing how to place a widget anywhere in the `Ui`:
         match *widget_type {
             WidgetType::Button => {
-                ui.put(widget_rect, Button::new("Example button"));
+                ui.put(widget_rect, egui::Button::new("Example button"));
             }
             WidgetType::Label => {
-                ui.put(widget_rect, Label::new("Example label"));
+                ui.put(widget_rect, egui::Label::new("Example label"));
             }
             WidgetType::TextEdit => {
-                ui.put(widget_rect, TextEdit::multiline(text_edit_contents));
+                ui.put(widget_rect, egui::TextEdit::multiline(text_edit_contents));
             }
         }
     }

egui_demo_lib/src/apps/demo/toggle_switch.rs

@@ -30,6 +30,11 @@ pub fn toggle_ui(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
     // This is where we get a region of the screen assigned.
     // We also tell the Ui to sense clicks in the allocated region.
     let (rect, mut response) = ui.allocate_exact_size(desired_size, egui::Sense::click());
+    if response.gained_kb_focus() {
+        // Inform accessibility systems that the widget is selected:
+        ui.output()
+            .push_gained_focus_event(egui::WidgetType::Checkbox, "");
+    }
 
     // 3. Interact: Time to check for clicks!
     if response.clicked() {
@@ -67,6 +72,10 @@ pub fn toggle_ui(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
 fn toggle_ui_compact(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
     let desired_size = ui.spacing().interact_size.y * egui::vec2(2.0, 1.0);
     let (rect, mut response) = ui.allocate_exact_size(desired_size, egui::Sense::click());
+    if response.gained_kb_focus() {
+        ui.output()
+            .push_gained_focus_event(egui::WidgetType::Checkbox, "");
+    }
     if response.clicked() {
         *on = !*on;
         response.mark_changed();

egui_demo_lib/src/wrap_app.rs

@@ -117,6 +117,7 @@ impl epi::App for WrapApp {
         });
 
         self.backend_panel.update(ctx, frame);
+
         if self.backend_panel.open || ctx.memory().everything_is_visible() {
             egui::SidePanel::left("backend_panel", 150.0).show(ctx, |ui| {
                 self.backend_panel.ui(ui, frame);
@@ -128,6 +129,8 @@ impl epi::App for WrapApp {
                 app.update(ctx, frame);
             }
         }
+
+        self.backend_panel.end_of_frame(ctx);
     }
 }
 
@@ -216,6 +219,9 @@ struct BackendPanel {
 
     #[cfg_attr(feature = "persistence", serde(skip))]
     frame_history: crate::frame_history::FrameHistory,
+
+    #[cfg_attr(feature = "persistence", serde(skip))]
+    output_event_history: std::collections::VecDeque<egui::OutputEvent>,
 }
 
 impl Default for BackendPanel {
@@ -227,6 +233,7 @@ impl Default for BackendPanel {
             max_size_points_ui: egui::Vec2::new(1024.0, 2048.0),
             max_size_points_active: egui::Vec2::new(1024.0, 2048.0),
             frame_history: Default::default(),
+            output_event_history: Default::default(),
         }
     }
 }
@@ -242,6 +249,15 @@ impl BackendPanel {
         }
     }
 
+    fn end_of_frame(&mut self, ctx: &egui::CtxRef) {
+        for event in &ctx.output().events {
+            self.output_event_history.push_back(event.clone());
+        }
+        while self.output_event_history.len() > 10 {
+            self.output_event_history.pop_front();
+        }
+    }
+
     fn ui(&mut self, ui: &mut egui::Ui, frame: &mut epi::Frame<'_>) {
         ui.heading("💻 Backend");
 
@@ -298,6 +314,14 @@ impl BackendPanel {
                 frame.quit();
             }
         }
+
+        ui.collapsing("Output events", |ui| {
+            ui.set_max_width(350.0);
+            ui.label("Recent output events from egui:");
+            for event in &self.output_event_history {
+                ui.monospace(format!("{:?}", event));
+            }
+        });
     }
 
     fn pixels_per_point_ui(

egui_web/src/lib.rs

@@ -233,6 +233,7 @@ pub fn handle_output(output: &egui::Output) {
         open_url,
         copied_text,
         needs_repaint: _, // handled elsewhere
+        events: _,        // we ignore these (TODO: accessibility screen reader)
     } = output;
 
     set_cursor_icon(*cursor_icon);