Add initial Markdown Output support

.gitignore

@@ -2,3 +2,4 @@
 /Cargo.lock
 *.log
 sandbox/
+.env

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "serpapi"
-version = "1.2.0"
+version = "1.2.1"
 edition = "2021"
 description = "Integrate search data into your rust application. This library is the official wrapper for SerpApi. SerpApi supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more."
 readme = "README.md"

README.md

@@ -11,7 +11,7 @@ Integrate search data into your AI workflow, RAG, fine-tuning, or Rust applicati
 To install in your Rust application, update Cargo.toml:
 
 ```sh
-serpapi="1.2.0"
+serpapi="1.2.1"
 ```
 
 ## Quick start
@@ -102,6 +102,33 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
 More examples are available below.
 
+### Response formats
+
+Use `search` for structured results decoded into a `serde_json::Value`:
+
+```rust
+let results = client.search(parameter).await?;
+```
+
+Use `md` for a token-efficient Markdown `String` optimized for LLMs and AI agents.
+It returns clean headings, links and tables behind a YAML frontmatter while using
+roughly half the tokens of the JSON output:
+
+```rust
+let markdown = client.md(parameter).await?;
+```
+
+Use `html` when you need the raw response from the search engine:
+
+```rust
+let raw_html = client.html(parameter).await?;
+```
+
+`md` and `html` always force the output format, so any `output` search parameter is ignored.
+Archived results are also available as Markdown with `client.search_archive_md(&id).await?`.
+
+Learn more about [SerpApi Markdown output](https://serpapi.com/markdown-output).
+
 #### Documentation
 
  * [Full documentation on SerpApi.com](https://serpapi.com)
@@ -118,7 +145,7 @@ default.insert("api_key".to_string(), "your_secret_key".to_string());
 let client = Client::new(default).unwrap();
 let mut parameter = HashMap::<String, String>::new();
 parameter.insert("q".to_string(), "coffee".to_string());
-let markdown = client.markdown(parameter).await.expect("request");
+let markdown = client.md(parameter).await.expect("request");
 ```
 
 It returns the search results formatted as Markdown.
@@ -167,6 +194,12 @@ println!("{}", archived_results);
 assert_eq!(archive_id, search_id);
 ```
 
+The same archived search is available as Markdown:
+
+```rust
+let markdown = client.search_archive_md(&id).await.expect("request");
+```
+
 ### Account API
 ```rust
 let client = Client::new(HashMap::<String, String>::new());
@@ -178,11 +211,13 @@ let account = client.account(parameter).await.expect("request");
 It returns your account information.
 
 ### Technical features
+- Search results as JSON with `search`, Markdown with `md`, or raw search engine HTML with `html`
 - Dynamic JSON decoding using Serde JSON
 - Asyncronous HTTP request handle method using tokio and reqwest
 - Async tests using Tokio
 
 ### Change log
+- 1.2.1: Remove `markdown()` which called a non-existent endpoint, use `client.md(parameter)` and `client.search_archive_md(&id)` to return search results formatted as Markdown.
 - 1.2.0: Add `markdown` method to return search results formatted as Markdown.
 - 1.1.0: Always reuse the same client object instead of creating a new one for each search.
   - This is a breaking change for the API because the client must be unwrapped in the main function.

src/lib.rs

@@ -5,4 +5,7 @@
 //! SerpApi.com enables to do localized search, leverage advanced search engine features and a lot more...
 //! A completed documentation is available at [SerpApi](https://serpapi.com).
 //!
+//! Search results are available as JSON with `search`, as [Markdown](https://serpapi.com/markdown-output)
+//! optimized for LLMs and AI agents with `md`, or as raw search engine HTML with `html`.
+//!
 pub mod serpapi;

src/serpapi.rs

@@ -44,7 +44,7 @@ impl Client {
     /// async fn main() {
     ///  let mut default = std::collections::HashMap::<String, String>::new();
     ///  default.insert("engine".to_string(), "google".to_string());
-    ///  default.insert("api_key".to_string(), "secret_api_key".to_string());
+    ///  default.insert("api_key".to_string(), std::env::var("SERPAPI_KEY").expect("$SERPAPI_KEY is not set"));
     ///  // initialize the serpapi client
     ///  let client = Client::new(default).unwrap();
     ///  let mut parameter = HashMap::<String, String>::new();
@@ -67,41 +67,47 @@ impl Client {
         Ok(results)
     }
 
-    // execute a search and return the result as raw HTML formatted as String
+    /// execute a search on serpapi.com
+    ///  and return the results as Markdown formatted as String.
+    ///  The Markdown output is optimized for LLMs and AI agents.
+    ///  It holds a YAML frontmatter followed by headings, links and tables
+    ///  using about half the tokens of the JSON output.
+    ///  see: https://serpapi.com/markdown-output
     /// # Arguments
-    /// * `parameter` html search parameter
+    ///  * `parameter` search parameter, the output is always set to md.
+    ///
     /// # Examples:
-    /// ```
+    /// ```no_run
     /// use std::collections::HashMap;
     /// use serpapi::serpapi::Client;
     ///
     /// #[tokio::main]
     /// async fn main() {
-    /// let mut default = HashMap::<String, String>::new();
-    /// default.insert("engine".to_string(), "google".to_string());
-    /// default.insert("api_key".to_string(), "secret_api_key".to_string());
-    /// // initialize the search engine
-    /// let client = Client::new(default).unwrap();
-    /// let mut parameter = HashMap::<String, String>::new();
-    /// parameter.insert("q".to_string(), "coffee".to_string());
-    /// parameter.insert("location".to_string(), "Austin, TX, Texas, United States".to_string());
-    /// let html = client.html(parameter).await.expect("request");
-    /// assert!(html.len() > 100);
+    ///  let mut default = HashMap::<String, String>::new();
+    ///  default.insert("engine".to_string(), "google".to_string());
+    ///  default.insert("api_key".to_string(), "secret_api_key".to_string());
+    ///  // initialize the serpapi client
+    ///  let client = Client::new(default).unwrap();
+    ///  let mut parameter = HashMap::<String, String>::new();
+    ///  parameter.insert("q".to_string(), "coffee".to_string());
+    ///  // md returns the search results as a Markdown String.
+    ///  let markdown = client.md(parameter).await.expect("request");
+    ///  assert!(markdown.starts_with("---"));
     /// }
     /// ```
-    pub async fn html(
+    pub async fn md(
         &self,
         parameter: HashMap<String, String>,
     ) -> Result<String, Box<dyn std::error::Error>> {
-        let body = self.get("/html", parameter).await?;
+        let body = self.text("/search", force_output(parameter, "md")).await?;
         Ok(body)
     }
 
-    /// execute a search and return the result as Markdown formatted as String
+    // execute a search and return the result as raw HTML formatted as String
     /// # Arguments
-    /// * `parameter` markdown search parameter
+    /// * `parameter` html search parameter, the output is always set to html.
     /// # Examples:
-    /// ```
+    /// ```no_run
     /// use std::collections::HashMap;
     /// use serpapi::serpapi::Client;
     ///
@@ -114,15 +120,18 @@ impl Client {
     /// let client = Client::new(default).unwrap();
     /// let mut parameter = HashMap::<String, String>::new();
     /// parameter.insert("q".to_string(), "coffee".to_string());
-    /// let markdown = client.markdown(parameter).await.expect("request");
-    /// assert!(markdown.len() > 100);
+    /// parameter.insert("location".to_string(), "Austin, TX, Texas, United States".to_string());
+    /// let html = client.html(parameter).await.expect("request");
+    /// assert!(html.len() > 100);
     /// }
     /// ```
-    pub async fn markdown(
+    pub async fn html(
         &self,
         parameter: HashMap<String, String>,
     ) -> Result<String, Box<dyn std::error::Error>> {
-        let body = self.get("/markdown", parameter).await?;
+        let body = self
+            .text("/search", force_output(parameter, "html"))
+            .await?;
         Ok(body)
     }
 
@@ -165,6 +174,21 @@ impl Client {
         Ok(results)
     }
 
+    /// Retrieve a search result from the Search Archive API as Markdown.
+    ///  see: https://serpapi.com/markdown-output
+    /// # Arguments
+    /// * `search_id` from the original search: `results["search_metadata"]["id"]`
+    pub async fn search_archive_md(
+        &self,
+        search_id: &str,
+    ) -> Result<String, Box<dyn std::error::Error>> {
+        let mut endpoint = "/searches/".to_string();
+        endpoint.push_str(search_id);
+        endpoint.push_str(".md");
+        let body = self.text(&endpoint, HashMap::new()).await?;
+        Ok(body)
+    }
+
     // Get account information using Account API
     pub async fn account(
         &self,
@@ -181,15 +205,44 @@ impl Client {
     ) -> Result<serde_json::Value, Box<dyn std::error::Error>> {
         let body = self.get(endpoint, parameter).await?;
         //debug: println!("Body:\n{}", body);
-        let value: serde_json::Value = serde_json::from_str(&body).unwrap();
+        //  a non JSON body means the output is html or md, see: Client::html and Client::md
+        let value: serde_json::Value = serde_json::from_str(&body)?;
         Ok(value)
     }
 
+    /// execute a request and return the body as String.
+    ///  SerpApi reports errors as JSON even when html or md output is requested,
+    ///  so a JSON response to a text request is reported as an error.
+    /// # Arguments
+    /// * `endpoint` HTTP service URI
+    /// * `parameter` search parameter
+    pub async fn text(
+        &self,
+        endpoint: &str,
+        parameter: HashMap<String, String>,
+    ) -> Result<String, Box<dyn std::error::Error>> {
+        let (content_type, body) = self.raw(endpoint, parameter).await?;
+        if content_type.starts_with("application/json") {
+            return Err(format!("search failed on {} with: {}", endpoint, body).into());
+        }
+        Ok(body)
+    }
+
     pub async fn get(
         &self,
         endpoint: &str,
         parameter: HashMap<String, String>,
     ) -> Result<String, Box<dyn std::error::Error>> {
+        let (_content_type, body) = self.raw(endpoint, parameter).await?;
+        Ok(body)
+    }
+
+    /// execute a request and return the content type along with the body.
+    async fn raw(
+        &self,
+        endpoint: &str,
+        parameter: HashMap<String, String>,
+    ) -> Result<(String, String), Box<dyn std::error::Error>> {
         let mut query = HashMap::<String, String>::new();
         query.insert("source".to_string(), "rust".to_string());
         for (key, value) in self.parameter.iter() {
@@ -204,7 +257,24 @@ impl Client {
         let mut url = HOST.to_string();
         url.push_str(endpoint);
         let res = self.http.get(url).query(&query).send().await?;
+        let status = res.status();
+        let content_type = res
+            .headers()
+            .get(reqwest::header::CONTENT_TYPE)
+            .and_then(|value| value.to_str().ok())
+            .unwrap_or("")
+            .to_lowercase();
         let body = res.text().await?;
-        Ok(body)
+        if !status.is_success() {
+            return Err(format!("{} failed with HTTP {}: {}", endpoint, status, body).into());
+        }
+        Ok((content_type, body))
     }
 }
+
+/// force the output format whatever the caller provides.
+///  the format drives the return type: json -> serde_json::Value, html / md -> String.
+fn force_output(mut parameter: HashMap<String, String>, format: &str) -> HashMap<String, String> {
+    parameter.insert("output".to_string(), format.to_string());
+    parameter
+}