Improve widget info output for potential screen readers

Part of https://github.com/emilk/egui/issues/167

egui/src/containers/collapsing_header.rs

@@ -206,11 +206,7 @@ impl CollapsingHeader {
         desired_size = desired_size.at_least(ui.spacing().interact_size);
         let (_, rect) = ui.allocate_space(desired_size);
 
-        let header_response = ui.interact(rect, id, Sense::click());
+        let mut header_response = ui.interact(rect, id, Sense::click());
-        if header_response.gained_kb_focus() {
-            ui.output()
-                .push_gained_focus_event(WidgetType::CollapsingHeader, &galley.text);
-        }
         let text_pos = pos2(
             text_pos.x,
             header_response.rect.center().y - galley.size.y / 2.0,
@@ -219,7 +215,10 @@ impl CollapsingHeader {
         let mut state = State::from_memory_with_default_open(ui.ctx(), id, default_open);
         if header_response.clicked() {
             state.toggle(ui);
+            header_response.mark_changed();
         }
+        header_response
+            .widget_info(|| WidgetInfo::labeled(WidgetType::CollapsingHeader, &galley.text));
 
         let visuals = ui.style().interact(&header_response);
         let text_color = visuals.text_color();

egui/src/containers/combo_box.rs

@@ -27,10 +27,7 @@ pub fn combo_box_with_label(
 
     ui.horizontal(|ui| {
         let mut response = combo_box(ui, button_id, selected, menu_contents);
-        if response.gained_kb_focus() {
+        response.widget_info(|| WidgetInfo::labeled(WidgetType::ComboBox, label.text()));
-            ui.output()
-                .push_gained_focus_event(WidgetType::ComboBox, label.text());
-        }
         response |= ui.add(label);
         response
     })

egui/src/data/output.rs

@@ -25,6 +25,14 @@ pub struct Output {
     pub events: Vec<OutputEvent>,
 }
 
+impl Output {
+    /// Open the given url in a web browser.
+    /// If egui is running in a browser, the same tab will be reused.
+    pub fn open_url(&mut self, url: impl Into<String>) {
+        self.open_url = Some(OpenUrl::new_tab(url))
+    }
+}
+
 #[derive(Clone, PartialEq)]
 pub struct OpenUrl {
     pub url: String,
@@ -77,20 +85,129 @@ impl Default for CursorIcon {
 /// Things that happened during this frame that the integration may be interested in.
 ///
 /// In particular, these events may be useful for accessability, i.e. for screen readers.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, PartialEq)]
 pub enum OutputEvent {
     /// A widget gained keyboard focus (by tab key).
-    ///
+    WidgetEvent(WidgetEvent, WidgetInfo),
-    /// An integration can for instance read the newly selected widget out loud for the visually impaired.
+}
-    //
+
-    // TODO: we should output state too, e.g. if a checkbox is selected, or current slider value.
+impl std::fmt::Debug for OutputEvent {
-    Focused(WidgetType, String),
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::WidgetEvent(we, wi) => write!(f, "{:?}: {:?}", we, wi),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum WidgetEvent {
+    /// Keyboard focused moved onto the widget.
+    Focus,
+    // /// Started hovering a new widget.
+    // Hover, // TODO: cursor hovered events
+}
+
+/// Describes a widget such as a [`crate::Button`] or a [`crate::TextEdit`].
+#[derive(Clone, PartialEq)]
+pub struct WidgetInfo {
+    /// The type of widget this is.
+    pub typ: WidgetType,
+    /// The text on labels, buttons, checkboxes etc.
+    pub label: Option<String>,
+    /// The contents of some editable text (for `TextEdit` fields).
+    pub edit_text: Option<String>,
+    /// The current value of checkboxes and radio buttons.
+    pub selected: Option<bool>,
+    /// The current value of sliders etc.
+    pub value: Option<f64>,
+}
+
+impl std::fmt::Debug for WidgetInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let Self {
+            typ,
+            label,
+            edit_text,
+            selected,
+            value,
+        } = self;
+
+        let mut s = f.debug_struct("WidgetInfo");
+
+        s.field("typ", typ);
+
+        if let Some(label) = label {
+            s.field("label", label);
+        }
+        if let Some(edit_text) = edit_text {
+            s.field("edit_text", edit_text);
+        }
+        if let Some(selected) = selected {
+            s.field("selected", selected);
+        }
+        if let Some(value) = value {
+            s.field("value", value);
+        }
+
+        s.finish()
+    }
+}
+
+impl WidgetInfo {
+    pub fn new(typ: WidgetType) -> Self {
+        Self {
+            typ,
+            label: None,
+            edit_text: None,
+            selected: None,
+            value: None,
+        }
+    }
+
+    pub fn labeled(typ: WidgetType, label: impl Into<String>) -> Self {
+        Self {
+            label: Some(label.into()),
+            ..Self::new(typ)
+        }
+    }
+
+    /// checkboxes, radio-buttons etc
+    pub fn selected(typ: WidgetType, selected: bool, label: impl Into<String>) -> Self {
+        Self {
+            label: Some(label.into()),
+            selected: Some(selected),
+            ..Self::new(typ)
+        }
+    }
+
+    pub fn drag_value(value: f64) -> Self {
+        Self {
+            value: Some(value),
+            ..Self::new(WidgetType::DragValue)
+        }
+    }
+
+    pub fn slider(value: f64, label: impl Into<String>) -> Self {
+        let label = label.into();
+        Self {
+            label: if label.is_empty() { None } else { Some(label) },
+            value: Some(value),
+            ..Self::new(WidgetType::Slider)
+        }
+    }
+
+    pub fn text_edit(edit_text: impl Into<String>) -> Self {
+        Self {
+            edit_text: Some(edit_text.into()),
+            ..Self::new(WidgetType::TextEdit)
+        }
+    }
 }
 
 /// The different types of built-in widgets in egui
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub enum WidgetType {
-    Label,
+    Label, // TODO: emit Label events
     Hyperlink,
     TextEdit,
     Button,
@@ -103,18 +220,9 @@ pub enum WidgetType {
     ColorButton,
     ImageButton,
     CollapsingHeader,
-}
-
-impl Output {
-    /// Open the given url in a web browser.
-    /// If egui is running in a browser, the same tab will be reused.
-    pub fn open_url(&mut self, url: impl Into<String>) {
-        self.open_url = Some(OpenUrl::new_tab(url))
-    }
 
-    /// Inform the backend integration that a widget gained focus
+    /// If you cannot fit any of the above slots.
-    pub fn push_gained_focus_event(&mut self, widget_type: WidgetType, text: impl Into<String>) {
+    ///
-        self.events
+    /// If this is something you think should be added, file an issue.
-            .push(OutputEvent::Focused(widget_type, text.into()));
+    Other,
-    }
 }

egui/src/lib.rs

@@ -309,7 +309,7 @@ pub use {
     context::{Context, CtxRef},
     data::{
         input::*,
-        output::{self, CursorIcon, Output, WidgetType},
+        output::{self, CursorIcon, Output, WidgetInfo, WidgetType},
     },
     grid::Grid,
     id::Id,

egui/src/response.rs

@@ -335,6 +335,18 @@ impl Response {
         let scroll_target = lerp(self.rect.y_range(), align.to_factor());
         self.ctx.frame_state().scroll_target = Some((scroll_target, align));
     }
+
+    /// For accessibility.
+    ///
+    /// Call after interacting and potential calls to [`Self::mark_changed`].
+    pub fn widget_info(&self, make_info: impl Fn() -> crate::WidgetInfo) {
+        if self.gained_kb_focus() {
+            use crate::output::{OutputEvent, WidgetEvent};
+            let widget_info = make_info();
+            let event = OutputEvent::WidgetEvent(WidgetEvent::Focus, widget_info);
+            self.ctx.output().events.push(event);
+        }
+    }
 }
 
 impl Response {

egui/src/widgets/button.rs

@@ -119,10 +119,7 @@ impl Button {
         }
 
         let (rect, response) = ui.allocate_at_least(desired_size, sense);
-        if response.gained_kb_focus() {
+        response.widget_info(|| WidgetInfo::labeled(WidgetType::Button, &galley.text));
-            ui.output()
-                .push_gained_focus_event(WidgetType::TextEdit, &galley.text);
-        }
 
         if ui.clip_rect().intersects(rect) {
             let visuals = ui.style().interact(&response);
@@ -232,15 +229,12 @@ impl<'a> Widget for Checkbox<'a> {
         desired_size = desired_size.at_least(spacing.interact_size);
         desired_size.y = desired_size.y.max(icon_width);
         let (rect, mut response) = ui.allocate_exact_size(desired_size, Sense::click());
-        if response.gained_kb_focus() {
-            ui.output()
-                .push_gained_focus_event(WidgetType::Checkbox, &galley.text);
-        }
 
         if response.clicked() {
             *checked = !*checked;
             response.mark_changed();
         }
+        response.widget_info(|| WidgetInfo::selected(WidgetType::Checkbox, *checked, &galley.text));
 
         // let visuals = ui.style().interact_selectable(&response, *checked); // too colorful
         let visuals = ui.style().interact(&response);
@@ -346,10 +340,8 @@ impl Widget for RadioButton {
         desired_size = desired_size.at_least(ui.spacing().interact_size);
         desired_size.y = desired_size.y.max(icon_width);
         let (rect, response) = ui.allocate_exact_size(desired_size, Sense::click());
-        if response.gained_kb_focus() {
+        response
-            ui.output()
+            .widget_info(|| WidgetInfo::selected(WidgetType::RadioButton, checked, &galley.text));
-                .push_gained_focus_event(WidgetType::RadioButton, &galley.text);
-        }
 
         let text_cursor = pos2(
             rect.min.x + button_padding.x + icon_width + icon_spacing,
@@ -454,10 +446,7 @@ impl Widget for ImageButton {
         let button_padding = ui.spacing().button_padding;
         let size = image.size() + 2.0 * button_padding;
         let (rect, response) = ui.allocate_exact_size(size, sense);
-        if response.gained_kb_focus() {
+        response.widget_info(|| WidgetInfo::new(WidgetType::ImageButton));
-            ui.output()
-                .push_gained_focus_event(WidgetType::ImageButton, "");
-        }
 
         if ui.clip_rect().intersects(rect) {
             let visuals = ui.style().interact(&response);

egui/src/widgets/color_picker.rs

@@ -66,10 +66,7 @@ fn show_hsva(ui: &mut Ui, color: Hsva, desired_size: Vec2) -> Response {
 fn color_button(ui: &mut Ui, color: Color32) -> Response {
     let size = ui.spacing().interact_size;
     let (rect, response) = ui.allocate_exact_size(size, Sense::click());
-    if response.gained_kb_focus() {
+    response.widget_info(|| WidgetInfo::new(WidgetType::ColorButton));
-        ui.output()
-            .push_gained_focus_event(WidgetType::ColorButton, "");
-    }
     let visuals = ui.style().interact(&response);
     let rect = rect.expand(visuals.expansion);
 

egui/src/widgets/drag_value.rs

@@ -298,16 +298,12 @@ impl<'a> Widget for DragValue<'a> {
             response
         };
 
-        if response.gained_kb_focus() {
-            ui.output()
-                .push_gained_focus_event(WidgetType::DragValue, "");
-        }
-
         #[allow(clippy::float_cmp)]
         {
             response.changed = get(&mut get_set_value) != value;
         }
 
+        response.widget_info(|| WidgetInfo::drag_value(value));
         response
     }
 }

egui/src/widgets/hyperlink.rs

@@ -53,10 +53,7 @@ impl Widget for Hyperlink {
         let Hyperlink { url, label } = self;
         let galley = label.layout(ui);
         let (rect, response) = ui.allocate_exact_size(galley.size, Sense::click());
-        if response.gained_kb_focus() {
+        response.widget_info(|| WidgetInfo::labeled(WidgetType::Hyperlink, &galley.text));
-            ui.output()
-                .push_gained_focus_event(WidgetType::Hyperlink, &galley.text);
-        }
 
         if response.hovered() {
             ui.ctx().output().cursor_icon = CursorIcon::PointingHand;

egui/src/widgets/selected_label.rs

@@ -55,10 +55,9 @@ impl Widget for SelectableLabel {
         let mut desired_size = total_extra + galley.size;
         desired_size.y = desired_size.y.at_least(ui.spacing().interact_size.y);
         let (rect, response) = ui.allocate_at_least(desired_size, Sense::click());
-        if response.gained_kb_focus() {
+        response.widget_info(|| {
-            ui.output()
+            WidgetInfo::selected(WidgetType::SelectableLabel, selected, &galley.text)
-                .push_gained_focus_event(WidgetType::SelectableLabel, &galley.text);
+        });
-        }
 
         let text_cursor = ui
             .layout()

egui/src/widgets/slider.rs

@@ -322,10 +322,8 @@ impl<'a> Slider<'a> {
             self.set_value(new_value);
         }
 
-        if response.gained_kb_focus() {
+        let value = self.get_value();
-            ui.output()
+        response.widget_info(|| WidgetInfo::slider(value, &self.text));
-                .push_gained_focus_event(WidgetType::Slider, &self.text);
-        }
 
         if response.has_kb_focus() {
             let kb_step = ui.input().num_presses(Key::ArrowRight) as f32

egui/src/widgets/text_edit.rs

@@ -321,10 +321,6 @@ impl<'t> TextEdit<'t> {
             Sense::hover()
         };
         let mut response = ui.interact(rect, id, sense);
-        if response.gained_kb_focus() {
-            ui.output()
-                .push_gained_focus_event(WidgetType::TextEdit, &*text);
-        }
 
         if enabled {
             if let Some(pointer_pos) = ui.input().pointer.interact_pos() {
@@ -523,6 +519,7 @@ impl<'t> TextEdit<'t> {
 
         ui.memory().text_edit.insert(id, state);
 
+        response.widget_info(|| WidgetInfo::text_edit(&*text));
         response
     }
 }

egui_demo_lib/src/apps/demo/toggle_switch.rs

@@ -30,11 +30,6 @@ pub fn toggle_ui(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
     // This is where we get a region of the screen assigned.
     // We also tell the Ui to sense clicks in the allocated region.
     let (rect, mut response) = ui.allocate_exact_size(desired_size, egui::Sense::click());
-    if response.gained_kb_focus() {
-        // Inform accessibility systems that the widget is selected:
-        ui.output()
-            .push_gained_focus_event(egui::WidgetType::Checkbox, "");
-    }
 
     // 3. Interact: Time to check for clicks!
     if response.clicked() {
@@ -42,6 +37,9 @@ pub fn toggle_ui(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
         response.mark_changed(); // report back that the value changed
     }
 
+    // Attach some meta-data to the response which can be used by screen readers:
+    response.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Checkbox, *on, ""));
+
     // 4. Paint!
     // First let's ask for a simple animation from egui.
     // egui keeps track of changes in the boolean associated with the id and
@@ -72,14 +70,11 @@ pub fn toggle_ui(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
 fn toggle_ui_compact(ui: &mut egui::Ui, on: &mut bool) -> egui::Response {
     let desired_size = ui.spacing().interact_size.y * egui::vec2(2.0, 1.0);
     let (rect, mut response) = ui.allocate_exact_size(desired_size, egui::Sense::click());
-    if response.gained_kb_focus() {
-        ui.output()
-            .push_gained_focus_event(egui::WidgetType::Checkbox, "");
-    }
     if response.clicked() {
         *on = !*on;
         response.mark_changed();
     }
+    response.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Checkbox, *on, ""));
 
     let how_on = ui.ctx().animate_bool(response.id, *on);
     let visuals = ui.style().interact_selectable(&response, *on);

egui_demo_lib/src/wrap_app.rs

@@ -316,10 +316,11 @@ impl BackendPanel {
         }
 
         ui.collapsing("Output events", |ui| {
-            ui.set_max_width(350.0);
+            ui.set_max_width(450.0);
             ui.label("Recent output events from egui:");
+            ui.advance_cursor(8.0);
             for event in &self.output_event_history {
-                ui.monospace(format!("{:?}", event));
+                ui.label(format!("{:?}", event));
             }
         });
     }