Generalize http fetch (#488)

* Generalize http fetch

- allow bytes as request body
- expose request and response headers in API
- update http example to show response headers and allow POST requests

* clippy fixes

* add missing comment, pub

* doc comment fix

* fix: missing argument when feature syntect not enabled

* formatting fixes

Co-authored-by: Emil Ernerfeldt <emil.ernerfeldt@gmail.com>

* remove commented out code

Co-authored-by: Emil Ernerfeldt <emil.ernerfeldt@gmail.com>

* formatting fixes

Co-authored-by: Emil Ernerfeldt <emil.ernerfeldt@gmail.com>

* cargo fmt

Co-authored-by: Emil Ernerfeldt <emil.ernerfeldt@gmail.com>

egui_demo_lib/src/apps/http_app.rs

@@ -1,10 +1,12 @@
-use epi::http::Response;
+use epi::http::{Request, Response};
 use std::sync::mpsc::Receiver;
 
 struct Resource {
     /// HTTP response
     response: Response,
 
+    text: Option<String>,
+
     /// If set, the response was an image.
     image: Option<Image>,
 
@@ -14,26 +16,43 @@ struct Resource {
 
 impl Resource {
     fn from_response(response: Response) -> Self {
-        let image = if response.header_content_type.starts_with("image/") {
+        let content_type = response.content_type().unwrap_or_default();
+        let image = if content_type.starts_with("image/") {
             Image::decode(&response.bytes)
         } else {
             None
         };
 
-        let colored_text = syntax_highlighting(&response);
+        let text = response.text();
+
+        let colored_text = text
+            .as_ref()
+            .and_then(|text| syntax_highlighting(&response, text));
 
         Self {
             response,
+            text,
             image,
             colored_text,
         }
     }
 }
 
+#[derive(Debug, PartialEq, Copy, Clone)]
+#[cfg_attr(feature = "persistence", derive(serde::Deserialize, serde::Serialize))]
+enum Method {
+    Get,
+    Post,
+}
+
 #[cfg_attr(feature = "persistence", derive(serde::Deserialize, serde::Serialize))]
 pub struct HttpApp {
     url: String,
 
+    method: Method,
+
+    request_body: String,
+
     #[cfg_attr(feature = "persistence", serde(skip))]
     in_progress: Option<Receiver<Result<Response, String>>>,
 
@@ -48,6 +67,8 @@ impl Default for HttpApp {
     fn default() -> Self {
         Self {
             url: "https://raw.githubusercontent.com/emilk/egui/master/README.md".to_owned(),
+            method: Method::Get,
+            request_body: r#"["posting some json", { "more_json" : true }]"#.to_owned(),
             in_progress: Default::default(),
             result: Default::default(),
             tex_mngr: Default::default(),
@@ -78,12 +99,18 @@ impl epi::App for HttpApp {
                 "(source code)"
             ));
 
-            if let Some(url) = ui_url(ui, frame, &mut self.url) {
+            if let Some(request) = ui_url(
+                ui,
+                frame,
+                &mut self.url,
+                &mut self.method,
+                &mut self.request_body,
+            ) {
                 let repaint_signal = frame.repaint_signal();
                 let (sender, receiver) = std::sync::mpsc::channel();
                 self.in_progress = Some(receiver);
 
-                frame.http_fetch(epi::http::Request::get(url), move |response| {
+                frame.http_fetch(request, move |response| {
                     sender.send(response).ok();
                     repaint_signal.request_repaint();
                 });
@@ -100,7 +127,10 @@ impl epi::App for HttpApp {
                     }
                     Err(error) => {
                         // This should only happen if the fetch API isn't available or something similar.
-                        ui.add(egui::Label::new(error).text_color(egui::Color32::RED));
+                        ui.add(
+                            egui::Label::new(if error.is_empty() { "Error" } else { error })
+                                .text_color(egui::Color32::RED),
+                        );
                     }
                 }
             }
@@ -108,13 +138,38 @@ impl epi::App for HttpApp {
     }
 }
 
-fn ui_url(ui: &mut egui::Ui, frame: &mut epi::Frame<'_>, url: &mut String) -> Option<String> {
+fn ui_url(
+    ui: &mut egui::Ui,
+    frame: &mut epi::Frame<'_>,
+    url: &mut String,
+    method: &mut Method,
+    request_body: &mut String,
+) -> Option<Request> {
     let mut trigger_fetch = false;
 
-    ui.horizontal(|ui| {
+    egui::Grid::new("request_params").show(ui, |ui| {
         ui.label("URL:");
-        trigger_fetch |= ui.text_edit_singleline(url).lost_focus();
-        trigger_fetch |= ui.button("GET").clicked();
+        ui.horizontal(|ui| {
+            trigger_fetch |= ui.text_edit_singleline(url).lost_focus();
+            egui::ComboBox::from_id_source("method")
+                .selected_text(format!("{:?}", method))
+                .width(60.0)
+                .show_ui(ui, |ui| {
+                    ui.selectable_value(method, Method::Get, "GET");
+                    ui.selectable_value(method, Method::Post, "POST");
+                });
+            trigger_fetch |= ui.button("▶").clicked();
+        });
+        ui.end_row();
+        if *method == Method::Post {
+            ui.label("Body:");
+            ui.add(
+                egui::TextEdit::multiline(request_body)
+                    .code_editor()
+                    .desired_rows(1),
+            );
+            ui.end_row();
+        }
     });
 
     if frame.is_web() {
@@ -127,6 +182,7 @@ fn ui_url(ui: &mut egui::Ui, frame: &mut epi::Frame<'_>, url: &mut String) -> Op
                 "https://raw.githubusercontent.com/emilk/egui/master/{}",
                 file!()
             );
+            *method = Method::Get;
             trigger_fetch = true;
         }
         if ui.button("Random image").clicked() {
@@ -134,12 +190,21 @@ fn ui_url(ui: &mut egui::Ui, frame: &mut epi::Frame<'_>, url: &mut String) -> Op
             let width = 640;
             let height = 480;
             *url = format!("https://picsum.photos/seed/{}/{}/{}", seed, width, height);
+            *method = Method::Get;
+            trigger_fetch = true;
+        }
+        if ui.button("Post to httpbin.org").clicked() {
+            *url = "https://httpbin.org/post".to_owned();
+            *method = Method::Post;
             trigger_fetch = true;
         }
     });
 
     if trigger_fetch {
-        Some(url.clone())
+        Some(match *method {
+            Method::Get => Request::get(url),
+            Method::Post => Request::post(url, request_body),
+        })
     } else {
         None
     }
@@ -153,6 +218,7 @@ fn ui_resource(
 ) {
     let Resource {
         response,
+        text,
         image,
         colored_text,
     } = resource;
@@ -162,13 +228,34 @@ fn ui_resource(
         "status:       {} ({})",
         response.status, response.status_text
     ));
-    ui.monospace(format!("Content-Type: {}", response.header_content_type));
     ui.monospace(format!(
-        "Size:         {:.1} kB",
+        "content-type: {}",
+        response.content_type().unwrap_or_default()
+    ));
+    ui.monospace(format!(
+        "size:         {:.1} kB",
         response.bytes.len() as f32 / 1000.0
     ));
 
-    if let Some(text) = &response.text {
+    ui.separator();
+
+    egui::CollapsingHeader::new("Response headers")
+        .default_open(false)
+        .show(ui, |ui| {
+            egui::Grid::new("response_headers")
+                .spacing(egui::vec2(ui.spacing().item_spacing.x * 2.0, 0.0))
+                .show(ui, |ui| {
+                    for header in &response.headers {
+                        ui.label(header.0);
+                        ui.label(header.1);
+                        ui.end_row();
+                    }
+                })
+        });
+
+    ui.separator();
+
+    if let Some(text) = &text {
         let tooltip = "Click to copy the response body";
         if ui.button("📋").on_hover_text(tooltip).clicked() {
             ui.output().copied_text = text.clone();
@@ -185,7 +272,7 @@ fn ui_resource(
             }
         } else if let Some(colored_text) = colored_text {
             colored_text.ui(ui);
-        } else if let Some(text) = &response.text {
+        } else if let Some(text) = &text {
             ui.monospace(text);
         } else {
             ui.monospace("[binary]");
@@ -197,8 +284,7 @@ fn ui_resource(
 // Syntax highlighting:
 
 #[cfg(feature = "syntect")]
-fn syntax_highlighting(response: &Response) -> Option<ColoredText> {
-    let text = response.text.as_ref()?;
+fn syntax_highlighting(response: &Response, text: &str) -> Option<ColoredText> {
     let extension_and_rest: Vec<&str> = response.url.rsplitn(2, '.').collect();
     let extension = extension_and_rest.get(0)?;
     ColoredText::text_with_extension(text, extension)
@@ -253,7 +339,7 @@ impl ColoredText {
 }
 
 #[cfg(not(feature = "syntect"))]
-fn syntax_highlighting(_: &Response) -> Option<ColoredText> {
+fn syntax_highlighting(_: &Response, _: &str) -> Option<ColoredText> {
     None
 }
 #[cfg(not(feature = "syntect"))]

egui_glium/src/http.rs

@@ -1,16 +1,20 @@
+use std::collections::BTreeMap;
+
 pub use epi::http::{Request, Response};
 
 /// NOTE: Ok(..) is returned on network error.
 /// Err is only for failure to use the fetch api.
 pub fn fetch_blocking(request: &Request) -> Result<Response, String> {
-    let Request { method, url, body } = request;
+    let mut req = ureq::request(&request.method, &request.url);
+
+    for header in &request.headers {
+        req = req.set(header.0, header.1);
+    }
 
-    let req = ureq::request(method, url).set("Accept", "*/*");
-    let resp = if body.is_empty() {
+    let resp = if request.body.is_empty() {
         req.call()
     } else {
-        req.set("Content-Type", "text/plain; charset=utf-8")
-            .send_string(body)
+        req.send_bytes(&request.body)
     };
 
     let (ok, resp) = match resp {
@@ -22,7 +26,13 @@ pub fn fetch_blocking(request: &Request) -> Result<Response, String> {
     let url = resp.get_url().to_owned();
     let status = resp.status();
     let status_text = resp.status_text().to_owned();
-    let header_content_type = resp.header("Content-Type").unwrap_or_default().to_owned();
+    let mut headers = BTreeMap::new();
+    for key in &resp.headers_names() {
+        if let Some(value) = resp.header(key) {
+            // lowercase for easy lookup
+            headers.insert(key.to_ascii_lowercase(), value.to_owned());
+        }
+    }
 
     let mut reader = resp.into_reader();
     let mut bytes = vec![];
@@ -31,22 +41,13 @@ pub fn fetch_blocking(request: &Request) -> Result<Response, String> {
         .read_to_end(&mut bytes)
         .map_err(|err| err.to_string())?;
 
-    let text = if header_content_type.starts_with("text")
-        || header_content_type == "application/javascript"
-    {
-        String::from_utf8(bytes.clone()).ok()
-    } else {
-        None
-    };
-
     let response = Response {
         url,
         ok,
         status,
         status_text,
-        header_content_type,
         bytes,
-        text,
+        headers,
     };
     Ok(response)
 }

egui_web/src/http.rs

@@ -7,67 +7,75 @@ pub use epi::http::{Request, Response};
 pub async fn fetch_async(request: &Request) -> Result<Response, String> {
     fetch_jsvalue(request)
         .await
-        .map_err(|err| err.as_string().unwrap_or_default())
+        .map_err(|err| err.as_string().unwrap_or(format!("{:#?}", err)))
 }
 
 /// NOTE: Ok(..) is returned on network error.
 /// Err is only for failure to use the fetch api.
 async fn fetch_jsvalue(request: &Request) -> Result<Response, JsValue> {
-    let Request { method, url, body } = request;
-
     // https://rustwasm.github.io/wasm-bindgen/examples/fetch.html
+    // https://github.com/seanmonstar/reqwest/blob/master/src/wasm/client.rs
 
     use wasm_bindgen::JsCast;
     use wasm_bindgen_futures::JsFuture;
 
     let mut opts = web_sys::RequestInit::new();
-    opts.method(method);
+    opts.method(&request.method);
     opts.mode(web_sys::RequestMode::Cors);
 
-    if !body.is_empty() {
-        opts.body(Some(&JsValue::from_str(body)));
+    if !request.body.is_empty() {
+        let body_bytes: &[u8] = &request.body;
+        let body_array: js_sys::Uint8Array = body_bytes.into();
+        let js_value: &JsValue = body_array.as_ref();
+        opts.body(Some(js_value));
     }
 
-    let request = web_sys::Request::new_with_str_and_init(url, &opts)?;
-    request.headers().set("Accept", "*/*")?;
-
-    let window = web_sys::window().unwrap();
-    let response = JsFuture::from(window.fetch_with_request(&request)).await?;
-    assert!(response.is_instance_of::<web_sys::Response>());
-    let response: web_sys::Response = response.dyn_into().unwrap();
+    let js_request = web_sys::Request::new_with_str_and_init(&request.url, &opts)?;
 
-    // // TODO: support binary get
+    for h in &request.headers {
+        js_request.headers().set(h.0, h.1)?;
+    }
 
-    // let body = JsFuture::from(response.text()?).await?;
-    // let body = body.as_string().unwrap_or_default();
+    let window = web_sys::window().unwrap();
+    let response = JsFuture::from(window.fetch_with_request(&js_request)).await?;
+    let response: web_sys::Response = response.dyn_into()?;
 
     let array_buffer = JsFuture::from(response.array_buffer()?).await?;
     let uint8_array = js_sys::Uint8Array::new(&array_buffer);
     let bytes = uint8_array.to_vec();
 
-    let header_content_type = response
-        .headers()
-        .get("Content-Type")
-        .ok()
-        .flatten()
-        .unwrap_or_default();
-
-    let text = if header_content_type.starts_with("text")
-        || header_content_type == "application/javascript"
-    {
-        String::from_utf8(bytes.clone()).ok()
-    } else {
-        None
-    };
+    // https://developer.mozilla.org/en-US/docs/Web/API/Headers
+    // "Note: When Header values are iterated over, [...] values from duplicate header names are combined."
+    let mut headers = std::collections::BTreeMap::<String, String>::new();
+    let js_headers: web_sys::Headers = response.headers();
+    let js_iter = js_sys::try_iter(&js_headers)
+        .expect("headers try_iter")
+        .expect("headers have an iterator");
+
+    for item in js_iter {
+        let item = item.expect("headers iterator");
+        let array: js_sys::Array = item.into();
+        let v: Vec<JsValue> = array.to_vec();
+
+        let mut key = v[0]
+            .as_string()
+            .ok_or_else(|| JsValue::from_str("headers name"))?;
+        let value = v[1]
+            .as_string()
+            .ok_or_else(|| JsValue::from_str("headers value"))?;
+
+        // for easy lookup
+        key.make_ascii_lowercase();
+        headers.insert(key, value);
+    }
 
     Ok(Response {
-        status_text: response.status_text(),
         url: response.url(),
         ok: response.ok(),
         status: response.status(),
-        header_content_type,
+        status_text: response.status_text(),
         bytes,
-        text,
+        headers,
     })
 }
 

epi/src/lib.rs

@@ -377,34 +377,50 @@ pub const APP_KEY: &str = "app";
 ///
 /// You must enable the "http" feature for this.
 pub mod http {
-    /// A simple http requests.
+    use std::collections::BTreeMap;
+
+    /// A simple http request.
     pub struct Request {
         /// "GET", …
         pub method: String,
         /// https://…
         pub url: String,
-        /// x-www-form-urlencoded body
-        pub body: String,
+        /// The raw bytes.
+        pub body: Vec<u8>,
+        /// ("Accept", "*/*"), …
+        pub headers: BTreeMap<String, String>,
     }
 
     impl Request {
-        /// Create a `GET` requests with the given url.
+        pub fn create_headers_map(headers: &[(&str, &str)]) -> BTreeMap<String, String> {
+            headers
+                .iter()
+                .map(|e| (e.0.to_owned(), e.1.to_owned()))
+                .collect()
+        }
+
+        /// Create a `GET` request with the given url.
         #[allow(clippy::needless_pass_by_value)]
         pub fn get(url: impl ToString) -> Self {
             Self {
                 method: "GET".to_owned(),
                 url: url.to_string(),
-                body: "".to_string(),
+                body: vec![],
+                headers: Request::create_headers_map(&[("Accept", "*/*")]),
             }
         }
 
-        /// Create a `POST` requests with the give url and body.
+        /// Create a `POST` request with the given url and body.
         #[allow(clippy::needless_pass_by_value)]
         pub fn post(url: impl ToString, body: impl ToString) -> Self {
             Self {
                 method: "POST".to_owned(),
                 url: url.to_string(),
-                body: body.to_string(),
+                body: body.to_string().into_bytes(),
+                headers: Request::create_headers_map(&[
+                    ("Accept", "*/*"),
+                    ("Content-Type", "text/plain; charset=utf-8"),
+                ]),
             }
         }
     }
@@ -419,18 +435,21 @@ pub mod http {
         pub status: u16,
         /// Status text (e.g. "File not found" for status code `404`).
         pub status_text: String,
-
-        /// Content-Type header, or empty string if missing.
-        pub header_content_type: String,
-
         /// The raw bytes.
         pub bytes: Vec<u8>,
 
-        /// UTF-8 decoded version of bytes.
-        /// ONLY if `header_content_type` starts with "text" and bytes is UTF-8.
-        pub text: Option<String>,
+        pub headers: BTreeMap<String, String>,
     }
 
+    impl Response {
+        pub fn text(&self) -> Option<String> {
+            String::from_utf8(self.bytes.clone()).ok()
+        }
+
+        pub fn content_type(&self) -> Option<String> {
+            self.headers.get("content-type").cloned()
+        }
+    }
     /// Possible errors does NOT include e.g. 404, which is NOT considered an error.
     pub type Error = String;
 }