lib.rs - source

//! Library for sanitizing and converting HTML strings to something that [Pango] can render.
//!
//! This library contains several functions to (pre)process text to [Pango Markup].
//! What to use and when depends on the type of input and the desired result. This can range from
//! just escaping to converting and sanitizing. See the examples below for what is available
//! based on the input type.
//!
//! The functions below convert strings to strings. If your input can contain several block
//! elements such as headings, lists, code or quote blocks, see the [`block`] module to convert an
//! input string into a list of these blocks.
//!
//! # Markdown/body HTML
//!
//! To handle more HTML, use the [`markup_html`] function. This function supports HTML body markup
//! such as HTML resulting from a [Markdown]-to-HTML conversion. It tries to convert the input to
//! [Pango Markup] such that rendering by Pango will make it similar like what a browser would.
//! This involves adding newlines for paragraphs and lists, converting font styles, etc.
//!
//! ```rust
//! # use html2pango::markup_html;
//! let m = markup_html("<body>this is some <font color=\"#ff0000\">red text</font>!</body>").unwrap();
//! assert_eq!(m, "this is some <span foreground=\"#ff0000\">red text</span>!");
//!
//! let m = markup_html("<body>a nice <a href=\"https://gnome.org\">link</a>").unwrap();
//! assert_eq!(m, "a nice <a href=\"https://gnome.org\">link</a>");
//!
//! let m = markup_html("<body>some items: <ul><li>first</li><li>second</li></ul><body").unwrap();
//! assert_eq!(m, "some items: \n • first\n • second\n");
//! ```
//!
//! ## Escaping
//!
//! To just escape any HTML reserved characters, use [`html_escape`]:
//!
//! ```rust
//! # use html2pango::html_escape;
//! let s = html_escape("this is a <tag> & this is \"quoted text\"");
//! assert_eq!(s, "this is a &lt;tag&gt; &amp; this is &quot;quoted text&quot;");
//! ```
//!
//! ## Matrix custom HTML
//!
//! For [Matrix], its [specification][Matrix specification] defines a custom HTML format that
//! specifies the tags and attributes that can be used. Use [`matrix_html_to_markup`] to handle
//! this custom HTML input so that input is sanitized before it is converted.
//!
//! This function is still work-in-progress!
//!
//! # Simple HTML
//!
//! By simple HTML, we mean plain text that only contains some formatting tags such as
//! `<strong>`, `<i>`, `<code>`, etc.
//! For the full list of supported tags and how they are replaced, see [`markup_from_raw`].
//!
//!
//! ## With sanitization
//!
//! If you use [`markup`], supported tags are replaced (if necessary), malformed tags are removed
//! and HTML reserved characters are escaped.
//!
//! ```rust
//! # use html2pango::markup;
//! let m = markup("<p><strong>this <i>is &sanitized<f;><unsupported/></i></strong></p>");
//! assert_eq!(m, "<b>this <i>is &amp;sanitized</i></b>");
//! ```
//!
//! Other unsupported, but valid tags are escaped.
//!
//! ```rust
//! # use html2pango::markup;
//! let m = markup("this is <span>a tag</span>");
//! assert_eq!(m, "this is &lt;span&gt;a tag&lt;/span&gt;");
//! ```
//!
//! URIs are replaced by links.
//!
//! ```rust
//! # use html2pango::markup;
//! let m = markup("go to: https://gnome.org");
//! assert_eq!(m, "go to: <a href=\"https://gnome.org\">https://gnome.org</a>");
//! ```
//!
//! ## Without sanitization
//!
//! Use [`markup_from_raw`] if you have already sanitized input:
//!
//! ```rust
//! # use html2pango::markup_from_raw;
//! let m = markup_from_raw("<p>this is <unsupported>already sanitized</unsupported></p>");
//! assert_eq!(m, "this is &lt;unsupported&gt;already sanitized&lt;/unsupported&gt;");
//! ```
//!
//! ## Links
//!
//! To just replace URIs by links, use [`markup_links`]:
//!
//! ```rust
//! # use html2pango::markup_links;
//! let m = markup_links("go to: https://gnome.org");
//! assert_eq!(m, "go to: <a href=\"https://gnome.org\">https://gnome.org</a>");
//! ```
//!
//! [Markdown]: https://daringfireball.net/projects/markdown/
//! [Matrix]: https://matrix.org/
//! [Matrix specification]: https://spec.matrix.org/v1.1/client-server-api/#mroommessage-msgtypes
//! [Pango]: https://pango.gnome.org
//! [Pango Markup]: https://docs.gtk.org/Pango/pango_markup.html

use html5ever::tendril::TendrilSink;
use html5ever::tree_builder::TreeBuilderOpts;
use html5ever::{parse_document, ParseOpts};
use markup5ever_rcdom::Node;
use markup5ever_rcdom::NodeData;
use markup5ever_rcdom::RcDom;

use linkify::{LinkFinder, LinkKind};
use maplit::{hashmap, hashset};
use once_cell::sync::Lazy;

use regex::{Captures, Regex};
use std::fmt::Write as _;

use std::borrow::Cow;

pub mod block;

/// Sanitizes and converts simple HTML markup to Pango Markup.
///
/// This will sanitize the input string using the [`ammonia`][ammonia] crate's defaults and then
/// use [`markup_from_raw`] to convert the sanitized string to [Pango Markup].
///
/// # Examples
///
/// ```rust
/// # use html2pango::markup;
/// let m = markup("this is parsed");
/// assert_eq!(m, "this is parsed");
///
/// let m = markup("<b>this <i>is &ssd<f;</i></b>");
/// assert_eq!(m, "<b>this <i>is &amp;ssd</i></b>");
///
/// let m = markup("this is <span>parsed</span>");
/// assert_eq!(m, "this is &lt;span&gt;parsed&lt;/span&gt;");
///
/// let m = markup("with links: http://gnome.org");
/// assert_eq!(m, "with links: <a href=\"http://gnome.org\">http://gnome.org</a>");
/// ```
///
/// [Pango Markup]: https://docs.gtk.org/Pango/pango_markup.html
/// [ammonia]: https://docs.rs/ammonia/3.1.2/ammonia/fn.clean.html
pub fn markup(s: &str) -> String {
    let sanitized_html = ammonia::Builder::new().link_rel(None).clean(s).to_string();
    markup_from_raw(&sanitized_html)
}

/// Converts simple HTML markup to Pango Markup.
///
/// This will convert the input string to [Pango Markup] and replace URIs by links.
///
/// Currently it support conversion of the following HTML tags:
/// * `<p>` and `</p>` => `""`
/// * `<i>`, `<em>` and `</i>`,` </em>` => `<i>` and `</i>`
/// * `<b>`, `<strong>` and `</b>`, `</strong>` => `<b>` and `</b>`
/// * `<code>` and `</code>` => `<tt>` and `</tt>`
/// * `<sub>` and `</sub>` => `<sub>` and `</sub>`
/// * `<sup>` and `</sup>` => `<sup>` and `</sup>`
/// * `<br>` => `\n`
///
/// It will also convert the following characters and HTML entities:
/// * `<` => `&lt`
/// * `>` => `&gt`
/// * `&nbsp;` => ` ` (non breaking space)
///
/// # Examples
///
/// ```
/// # use html2pango::markup_from_raw;
/// let m = markup_from_raw("this is parsed");
/// assert_eq!(m, "this is parsed");
///
/// let m = markup_from_raw("<b>this <i>is &ssd<f;</i></b>");
/// assert_eq!(m, "<b>this <i>is &ssd&lt;f;</i></b>");
///
/// let m = markup_from_raw("this is <span>parsed</span>");
/// assert_eq!(m, "this is &lt;span&gt;parsed&lt;/span&gt;");
///
/// let m = markup_from_raw("with links: http://gnome.org");
/// assert_eq!(m, "with links: <a href=\"http://gnome.org\">http://gnome.org</a>");
/// ```
///
/// [Pango Markup]: https://docs.gtk.org/Pango/pango_markup.html
pub fn markup_from_raw(s: &str) -> String {
    static TAGS: Lazy<Regex> = Lazy::new(|| {
        Regex::new(concat![
            r"<p>|</p>|",
            r"<br>|",
            r"<b>|</b>|",
            r"<strong>|</strong>|",
            r"<code>|</code>|",
            r"<i>|</i>|",
            r"<sub>|</sub>|",
            r"<sup>|</sup>|",
            r"<em>|</em>|",
            r"<|>|",
            r"&nbsp;|",
            // Url
            r"https?://",                       // Protocol
            r#"[^\s,)("]+"#,                    // Domain
            r"(\?((&amp;)?\w+(=[\w._-]+)?)*)*", // Parameters
            r"(#[\w._-]+)?",                    // Hash
        ])
        .unwrap()
    });

    TAGS.replace_all(s.trim(), |captures: &Captures<'_>| {
        let m = captures.get(0).unwrap().as_str();
        // Keep this in sync with the TAGS regex
        match m {
            "<p>" | "</p>" => Cow::Borrowed(""),
            "<br>" => Cow::Borrowed("\n"),
            "<b>" | "<strong>" => Cow::Borrowed("<b>"),
            "</b>" | "</strong>" => Cow::Borrowed("</b>"),
            "<code>" => Cow::Borrowed("<tt>"),
            "</code>" => Cow::Borrowed("</tt>"),
            "<i>" | "<em>" => Cow::Borrowed("<i>"),
            "</i>" | "</em>" => Cow::Borrowed("</i>"),
            "<sub>" => Cow::Borrowed("<sub>"),
            "</sub>" => Cow::Borrowed("</sub>"),
            "<sup>" => Cow::Borrowed("<sup>"),
            "</sup>" => Cow::Borrowed("</sup>"),
            "<" => Cow::Borrowed("&lt;"),
            ">" => Cow::Borrowed("&gt;"),
            "&nbsp;" => Cow::Borrowed(" "),
            _ => Cow::Owned(format!(r#"<a href="{}">{}</a>"#, m, m)),
        }
    })
    .into()
}

/// Sanitizes and convert Matrix custom HTML markup (formatted body) to Pango Markup.
///
/// First sanitizes the input string by leaving in only the HTML tags and attributes allowed
/// by the [Matrix specification] and then uses [`markup_html`] to convert the sanitized string to
/// [Pango Markup].
///
/// # Errors
///
/// Returns an error if reading from the strings fails.
///
/// # Examples
///
/// TODO
///
/// [Matrix specification]: https://spec.matrix.org/v1.1/client-server-api/#mroommessage-msgtypes
/// [Pango Markup]: https://docs.gtk.org/Pango/pango_markup.html
// WIP: only allow the html subset that matrix uses.
pub fn matrix_html_to_markup(s: &str) -> String {
    // https://github.com/matrix-org/matrix-react-sdk/blob/4bf5e44b2043bbe95faa66943878acad23dfb823/src/HtmlUtils.js#L178-L184
    #[rustfmt::skip]
    let allowed_tags = hashset![
        "font", // custom to matrix for IRC-style font coloring
        "del", // for markdown
        "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "p", "a", "ul", "ol", "sup", "sub",
        "nl", "li", "b", "i", "u", "strong", "em", "strike", "code", "hr", "br", "div",
        "table", "thead", "caption", "tbody", "tr", "th", "td", "pre", "span", "img",
    ];

    // https://github.com/matrix-org/matrix-react-sdk/blob/4bf5e44b2043bbe95faa66943878acad23dfb823/src/HtmlUtils.js#L185-L193
    let allowed_attributes = hashmap![
        // custom ones first:
        "font" => hashset!["color", "data-mx-bg-color", "data-mx-color", "style"],
        "span" => hashset!["data-mx-bg-color", "data-mx-color", "style"],
        // remote target: custom to matrix
        "a" => hashset!["href", "name", "target", "rel"],
        "img" => hashset!["src", "width", "height", "alt", "title"],
        "ol" => hashset!["start"],
        // We don't actually allow all classes, TODO: we should filter them afterwards
        "code" => hashset!["class"],
    ];

    // https://github.com/matrix-org/matrix-react-sdk/blob/4bf5e44b2043bbe95faa66943878acad23dfb823/src/HtmlUtils.js#L48
    let allowed_urls = hashset!["http", "https", "ftp", "mailto", "magnet"];

    let sanitized_html = ammonia::Builder::new()
        .url_schemes(allowed_urls)
        .tags(allowed_tags)
        .tag_attributes(allowed_attributes)
        .link_rel(None)
        .clean(s)
        .to_string();

    markup_html(&sanitized_html)
        .map(|x| x.trim_matches('\n').to_string())
        .unwrap_or_else(|_| markup_from_raw(&sanitized_html))
}

/// Escapes the HTML reserved characters.
///
/// This replaces the HTML reserved characters by their respective entities in the input string.
///
/// # Examples
///
/// ```rust
/// # use html2pango::html_escape;
/// let s = html_escape("this is a <tag>");
/// assert_eq!(s, "this is a &lt;tag&gt;");
///
/// let s = html_escape("he says: \"this & that\"");
/// assert_eq!(s, "he says: &quot;this &amp; that&quot;");
/// ```
pub fn html_escape(s: &str) -> String {
    s.to_string()
        .replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
        .replace('"', "&quot;")
}

/// Replaces URIs by HTML link tags.
///
/// This function assumes that HTML entities in input string have been escaped (see also:
/// [`html_escape`]). It will leave URIs in already existing links alone.
///
/// # Examples
///
/// ```rust
/// # use html2pango::{html_escape, markup_links};
/// let m = markup_links("go to https://gnome.org");
/// assert_eq!(m, "go to <a href=\"https://gnome.org\">https://gnome.org</a>");
/// ```
pub fn markup_links(s: &str) -> String {
    let mut parsed = String::with_capacity(s.len());
    let finder = LinkFinder::new();
    let mut prepend_str: Option<String> = None;

    for span in finder.spans(s) {
        let mut s = span.as_str().to_string();
        match span.kind() {
            Some(&LinkKind::Url) => {
                if s.ends_with("&amp") {
                    prepend_str = Some("&amp".to_string());
                    let t = s.len() - 4;
                    s.truncate(t);
                }
                if s.ends_with("&lt") {
                    prepend_str = Some("&lt".to_string());
                    let t = s.len() - 3;
                    s.truncate(t);
                }
                if s.ends_with("&gt") {
                    prepend_str = Some("&gt".to_string());
                    let t = s.len() - 3;
                    s.truncate(t);
                }
                if s.ends_with("&quot") {
                    prepend_str = Some("&quot".to_string());
                    let t = s.len() - 5;
                    s.truncate(t);
                }
                // This is to manage "> or "< or "&
                if s.ends_with("&quot;") {
                    prepend_str = Some("&quot;".to_string() + &prepend_str.unwrap_or_default());
                    let t = s.len() - 6;
                    s.truncate(t);
                }
                let _ = write!(parsed, "<a href=\"{0}\">{0}</a>", s);
            }
            _ => {
                if let Some(s) = prepend_str {
                    parsed.push_str(&s);
                    prepend_str = None;
                }
                parsed.push_str(&s);
            }
        };
    }

    parsed
}

fn convert_tag(t: &str) -> Option<(&str, &str)> {
    let allowed = [
        "a",
        "br",
        "em",
        "i",
        "p",
        "code",
        "strong",
        "b",
        "blockquote",
        "li",
        "sub",
        "sup",
    ];

    if !allowed.contains(&t) {
        return Some(("", ""));
    }

    match t {
        "em" | "i" => Some(("<i>", "</i>")),
        "blockquote" => Some(("<i>", "</i>")),
        "p" => Some(("\n", "\n")),
        "br" => Some(("\n", "")),
        "code" => Some(("<tt>", "</tt>")),
        "strong" | "b" => Some(("<b>", "</b>")),
        "li" => Some(("", "\n")),
        "sub" => Some(("<sub>", "</sub>")),
        "sup" => Some(("<sup>", "</sup>")),
        _ => None,
    }
}

fn match_tag(node: &Node, tags: &[&str]) -> bool {
    match node.data {
        NodeData::Element { name: ref n, .. } => {
            let node_tag = n.local.to_string();
            tags.contains(&node_tag.as_str())
        }
        _ => false,
    }
}

fn convert_node(node: &Node, autolinks: bool) -> String {
    let mut output = String::new();

    match node.data {
        NodeData::Text { contents: ref c } => {
            let escaped = &html_escape(&c.borrow().replace('\n', ""));
            if autolinks {
                output.push_str(&markup_links(escaped));
            } else {
                output.push_str(escaped);
            }
        }
        NodeData::Element {
            name: ref n,
            attrs: ref a,
            ..
        } => {
            let mut content = String::new();
            let tag = n.local.to_string();

            match &tag[..] {
                "ul" => {
                    content.push('\n');
                    for child in node.children.borrow().iter() {
                        if match_tag(child, &["li", "ul", "ol"]) {
                            let _ = write!(content, " • {}", &convert_node(child, true));
                        }
                    }
                }
                "ol" => {
                    let mut counter = 1;
                    content.push('\n');
                    for child in node.children.borrow().iter() {
                        if match_tag(child, &["li", "ul", "ol"]) {
                            let _ = write!(content, " {}. {}", counter, &convert_node(child, true));
                            counter += 1;
                        }
                    }
                }
                _ => {
                    for child in node.children.borrow().iter() {
                        content.push_str(&convert_node(child, true));
                    }
                }
            }

            match &tag[..] {
                "body" => {
                    output.push_str(&content);
                }
                "a" => {
                    let mut link = String::new();
                    let attrs = a.borrow();
                    for attr in attrs.iter() {
                        if &*attr.name.local == "href" {
                            link = attr.value.to_string();
                        }
                    }

                    let mut no_link_content = String::new();
                    for child in node.children.borrow().iter() {
                        no_link_content.push_str(&convert_node(child, false));
                    }

                    let _ = write!(
                        output,
                        "<a href=\"{}\">{}</a>",
                        html_escape(&link),
                        no_link_content
                    );
                }
                "font" => {
                    let attrs = a.borrow();
                    let span_attrs = attrs
                        .iter()
                        .flat_map(|attr| match &*attr.name.local {
                            "color" | "data-mx-color" => Some(("foreground", &*attr.value)),
                            "data-mx-bg-color" => Some(("background", &*attr.value)),
                            _ => None,
                        })
                        .collect::<Vec<(&str, &str)>>();

                    format_span(&mut output, content, span_attrs);
                }
                "span" => {
                    let attrs = a.borrow();
                    let span_attrs = attrs
                        .iter()
                        .flat_map(|attr| match &*attr.name.local {
                            "data-mx-color" => Some(("foreground", &*attr.value)),
                            "data-mx-bg-color" => Some(("background", &*attr.value)),
                            _ => None,
                        })
                        .collect::<Vec<(&str, &str)>>();

                    format_span(&mut output, content, span_attrs);
                }
                _ => {
                    if let Some((t1, t2)) = convert_tag(&tag) {
                        let _ = write!(output, "{}{}{}", t1, content, t2);
                    } else {
                        let _ = write!(output, "<{0}>{1}</{0}>", tag, content);
                    }
                }
            };
        }
        _ => {}
    }

    output
}

fn format_span(output: &mut String, content: String, attrs: Vec<(&str, &str)>) {
    if attrs.is_empty() {
        output.push_str(&content);
        return;
    }

    let _ = write!(output, "<span");
    for (attr, val) in attrs.into_iter() {
        let _ = write!(output, " {}=\"{}\"", attr, val);
    }
    let _ = write!(output, ">{}</span>", content);
}

/// Converts HTML body markup to Pango Markup.
///
/// While [`markup`]/[`markup_from_raw`] only supports simple HTML tags such as `<p>`, `<b>` and
/// `<br>`, this function tries to convert HTML body markup so that the resulting [Pango Markup]
/// will render to something that is similar to what a browser would render.
///
/// # Errors
///
/// Returns an error if reading from the strings fails.
///
/// # Examples
///
/// ```rust
/// # use html2pango::markup_html;
/// let m = markup_html("<body>this is some <font color=\"#ff0000\">red text</font>!</body>").unwrap();
/// assert_eq!(m, "this is some <span foreground=\"#ff0000\">red text</span>!");
///
/// let m = markup_html("<body>a nice <a href=\"https://gnome.org\">link</a>").unwrap();
/// assert_eq!(m, "a nice <a href=\"https://gnome.org\">link</a>");
///
/// let m = markup_html("<body>some items: <ul><li>first</li><li>second</li></ul><body").unwrap();
/// assert_eq!(m, "some items: \n • first\n • second\n");
/// ```
///
/// [Pango Markup]: https://docs.gtk.org/Pango/pango_markup.html
pub fn markup_html(s: &str) -> Result<String, anyhow::Error> {
    let opts = ParseOpts {
        tree_builder: TreeBuilderOpts {
            drop_doctype: true,
            ..Default::default()
        },
        ..Default::default()
    };
    let dom = parse_document(RcDom::default(), opts)
        .from_utf8()
        .read_from(&mut s.as_bytes())?;

    let document = &dom.document;
    let html = &document.children.borrow()[0];
    let body = &html.children.borrow()[1];

    Ok(convert_node(body, true))
}

#[cfg(test)]
mod test {
    use super::*;
    use pretty_assertions::assert_eq;

    #[test]
    fn test_markup() {
        let m = markup("this is parsed");
        assert_eq!(&m, "this is parsed");

        let m = markup("this is <span>parsed</span>");
        assert_eq!(&m, "this is &lt;span&gt;parsed&lt;/span&gt;");

        let m = markup("this is &ssdf;");
        assert_eq!(&m, "this is &amp;ssdf;");

        // TODO: add more tests
        let m = markup("<p>this <br>is &ssdf;</p>");
        assert_eq!(&m, "this \nis &amp;ssdf;");

        let m = markup("<b>this <i>is &ssd<f;</i></b>");
        assert_eq!(&m, "<b>this <i>is &amp;ssd</i></b>");

        let url = "http://url.com/test?foo1&foo2=test&foo3#hashing";
        let m = markup(&format!("this is &ssdf; {}", url));
        assert_eq!(
            &m,
            &format!(
                "this is &amp;ssdf; <a href=\"{0}\">{0}</a>",
                url.replace('&', "&amp;")
            )
        );

        for l in &[
            ("with links: http://gnome.org :D", "http://gnome.org"),
            (
                "with links: http://url.com/test.html&stuff :D",
                "http://url.com/test.html&stuff",
            ),
        ] {
            let m = markup(l.0);
            assert_eq!(
                &m,
                &format!(
                    "with links: <a href=\"{0}\">{0}</a> :D",
                    l.1.replace('&', "&amp;")
                )
            );
        }
    }

    #[test]
    // FIXME: Write specific tests instead of copying the above
    fn test_matrix() {
        let markup = matrix_html_to_markup;
        let m = markup("this is parsed");
        assert_eq!(&m, "this is parsed");

        let m = markup("this is <span>parsed</span>");
        assert_eq!(&m, "this is parsed");

        let m = markup("this is &ssdf;");
        assert_eq!(&m, "this is &amp;ssdf;");

        // TODO: add more tests
        let m = markup("<p>this <br>is &ssdf;</p>");
        assert_eq!(&m, "this \nis &amp;ssdf;");

        let m = markup("<b>this <i>is &ssd<f;</i></b>");
        assert_eq!(&m, "<b>this <i>is &amp;ssd</i></b>");

        let m = markup("hello <font color=\"#112233\">world</font>");
        assert_eq!(&m, "hello <span foreground=\"#112233\">world</span>");

        let m = markup(
            "hello <font data-mx-color=\"#112233\" data-mx-bg-color=\"#332211\">world</font>",
        );
        assert_eq!(
            &m,
            "hello <span foreground=\"#112233\" background=\"#332211\">world</span>"
        );

        let m = markup("hello <em><font color=\"#112233\">http://gnome.org</font></em>");
        assert_eq!(&m, "hello <i><span foreground=\"#112233\"><a href=\"http://gnome.org\">http://gnome.org</a></span></i>");

        let m = markup(
            "added <span data-mx-color=\"#112233\" data-mx-bg-color=\"#332211\">Label</span>",
        );
        assert_eq!(
            &m,
            "added <span foreground=\"#112233\" background=\"#332211\">Label</span>"
        );

        let url = "http://url.com/test?foo1&foo2=test&foo3#hashing";
        let m = markup(&format!("this is &ssdf; {}", url));
        assert_eq!(
            &m,
            &format!(
                "this is &amp;ssdf; <a href=\"{0}\">{0}</a>",
                url.replace('&', "&amp;")
            )
        );

        for l in &[
            ("with links: http://gnome.org :D", "http://gnome.org"),
            (
                "with links: http://url.com/test.html&stuff :D",
                "http://url.com/test.html&stuff",
            ),
        ] {
            let m = markup(l.0);
            assert_eq!(
                &m,
                &format!(
                    "with links: <a href=\"{0}\">{0}</a> :D",
                    l.1.replace('&', "&amp;")
                )
            );
        }
    }

    #[test]
    fn test_links() {
        let strings = [
            ("clean string without markup",
             "clean string without markup"),

            ("clean string with a <b>markup</b>",
             "clean string with a &lt;b&gt;markup&lt;/b&gt;"),

            ("clean string with a <b>markup</b> and link http://gnome.org/?p=1&q#hash",
             "clean string with a &lt;b&gt;markup&lt;/b&gt; and link <a href=\"http://gnome.org/?p=1&amp;q#hash\">http://gnome.org/?p=1&amp;q#hash</a>"),

            ("report-bug is: please report bugs with parabola packages on the packaging bug tracker at: https://labs.parabola.nu/projects/issue-tracker/issues?set_filter=1&tracker_id=1",
             "report-bug is: please report bugs with parabola packages on the packaging bug tracker at: <a href=\"https://labs.parabola.nu/projects/issue-tracker/issues?set_filter=1&amp;tracker_id=1\">https://labs.parabola.nu/projects/issue-tracker/issues?set_filter=1&amp;tracker_id=1</a>"),

             (
             "bill-auger, isacdaavid: there are two major issues I see with gnome-software. The first issue is that flathub, the largest repo for flatpaks, has nonfree software. If flathub isn't included by default, I think this is fine. The second is archlinux-appstream-data. The [PKGBUILD](https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/archlinux-appstream-data) does not use appstream-generator at all. However, it does require grabbing files from sources.archlinux.org",
             "bill-auger, isacdaavid: there are two major issues I see with gnome-software. The first issue is that flathub, the largest repo for flatpaks, has nonfree software. If flathub isn't included by default, I think this is fine. The second is archlinux-appstream-data. The [PKGBUILD](<a href=\"https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/archlinux-appstream-data\">https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/archlinux-appstream-data</a>) does not use appstream-generator at all. However, it does require grabbing files from sources.archlinux.org",
             ),

             ("links with problems: http://gnome.org/?p=1&",
              "links with problems: <a href=\"http://gnome.org/?p=1\">http://gnome.org/?p=1</a>&amp;"),
             ("links with problems: http://gnome.org/?p=1>",
              "links with problems: <a href=\"http://gnome.org/?p=1\">http://gnome.org/?p=1</a>&gt;"),
             ("links with problems: http://gnome.org/?p=1<",
              "links with problems: <a href=\"http://gnome.org/?p=1\">http://gnome.org/?p=1</a>&lt;"),
        ];

        for &(s, e) in strings.iter() {
            let m = markup_links(&html_escape(s));
            assert_eq!(&m, e);
        }
    }

    #[test]
    fn test_markup_links() {
        let strings = [
            ("This is a test message with <em>markdown</em><br /><a href=\"http://gnome.org\">gnome</a><br />and other link http://gnome.org",
             "This is a test message with <i>markdown</i>\n<a href=\"http://gnome.org\">gnome</a>\nand other link <a href=\"http://gnome.org\">http://gnome.org</a>"),
        ];

        for &(s, e) in strings.iter() {
            let m = markup_html(s).unwrap();
            assert_eq!(&m, e);
        }
    }

    #[test]
    fn test_ending_quote_link() {
        let strings = [
            ("<boxes:gnome-boxes xmlns:boxes=\"https://wiki.gnome.org/Apps/Boxes\">",
             "&lt;boxes:gnome-boxes xmlns:boxes=&quot;<a href=\"https://wiki.gnome.org/Apps/Boxes\">https://wiki.gnome.org/Apps/Boxes</a>&quot;&gt;"),
        ];

        for &(s, e) in strings.iter() {
            let m = markup_links(&html_escape(s));
            assert_eq!(&m, e);
        }
    }

    #[test]
    fn test_link_scape() {
        let strings = [
            ("<a href=\"https://forums.transbian.love/?page=thread&id=69\">https://forums.transbian.love/?page=thread&id=69</a>",
             "<a href=\"https://forums.transbian.love/?page=thread&amp;id=69\">https://forums.transbian.love/?page=thread&amp;id=69</a>"),
            ("<a href=\"https://forums.transbian.love/?page=thread&id=69\">https://forums.transbian.love/?page=thread&amp;id=69</a>",
             "<a href=\"https://forums.transbian.love/?page=thread&amp;id=69\">https://forums.transbian.love/?page=thread&amp;id=69</a>"),
            ("https://forums.transbian.love/?page=thread&id=69",
             "<a href=\"https://forums.transbian.love/?page=thread&amp;id=69\">https://forums.transbian.love/?page=thread&amp;id=69</a>"),
        ];

        for &(s, e) in strings.iter() {
            let m = markup_html(s).unwrap();
            assert_eq!(&m, e);
        }
    }

    #[test]
    fn test_superscript() {
        let strings = [("Une 2<sup>e</sup> entente", "Une 2<sup>e</sup> entente")];

        for &(s, e) in strings.iter() {
            let m = markup(s);
            assert_eq!(&m, e);
        }
    }

    #[test]
    fn test_subscript() {
        let strings = [("H<sub>2</sub>O", "H<sub>2</sub>O")];

        for &(s, e) in strings.iter() {
            let m = markup(s);
            assert_eq!(&m, e);
        }
    }
}