サンプルコードで身につける: 自己終了タグと script の中身 — 「適切な終了タグ」が来るまでは全部文字

#[derive(Debug, Clone, Copy, PartialEq)]
enum State {
    TagName,
    BeforeAttributeName,
    AttributeName,
    AfterAttributeName,
    SelfClosingStartTag, // 13.2.5.40 "/" のあと。次が ">" なら自己終了フラグを立てる
}

struct Attribute {
    name: String,
}

enum HtmlToken {
    StartTag { tag: String, self_closing: bool, attributes: Vec<Attribute> },
}

/// "<" の次から ">" までを読む(この回は値なし属性だけ)
fn read_start_tag(s: &str) -> HtmlToken {
    let chars: Vec<char> = s.chars().collect();
    let (mut tag, mut self_closing, mut attributes) = (String::new(), false, Vec::new());
    let mut state = State::TagName;
    let mut i = 0;
    while i < chars.len() {
        let c = chars[i];
        let mut reconsume = false;
        match state {
            State::TagName => match c {
                ' ' => state = State::BeforeAttributeName,
                '/' => state = State::SelfClosingStartTag,
                '>' => break,
                _ => tag.push(c.to_ascii_lowercase()),
            },
            State::BeforeAttributeName => match c {
                ' ' => {}
                '/' | '>' => {
                    state = State::AfterAttributeName;
                    reconsume = true;
                }
                _ => {
                    attributes.push(Attribute { name: String::new() });
                    state = State::AttributeName;
                    reconsume = true;
                }
            },
            State::AttributeName => match c {
                ' ' | '/' | '>' => {
                    state = State::AfterAttributeName;
                    reconsume = true;
                }
                _ => attributes.last_mut().unwrap().name.push(c.to_ascii_lowercase()),
            },
            State::AfterAttributeName => match c {
                ' ' => {}
                '/' => state = State::SelfClosingStartTag,
                '>' => break,
                _ => {
                    attributes.push(Attribute { name: String::new() });
                    state = State::AttributeName;
                    reconsume = true;
                }
            },
            State::SelfClosingStartTag => match c {
                '>' => {
                    self_closing = true; // "/>" でタグが閉じたときだけフラグが立つ
                    break;
                }
                _ => {
                    // "/" の次が ">" でない(例: "<br/ >")。"/" は無視して属性の前に戻る(parse error)
                    state = State::BeforeAttributeName;
                    reconsume = true;
                }
            },
        }
        if !reconsume {
            i += 1;
        }
    }
    HtmlToken::StartTag { tag, self_closing, attributes }
}

fn main() {
    for s in ["br>", "br/>", "br />", "input disabled/>", "br/ >", "div/>"].iter() {
        let HtmlToken::StartTag { tag, self_closing, attributes } = read_start_tag(s);
        let names: Vec<&str> = attributes.iter().map(|a| a.name.as_str()).collect();
        println!("<{s:<18} -> tag={tag} self_closing={self_closing} attributes={names:?}");
    }
}

use std::collections::VecDeque;

enum State {
    ScriptData,             // 13.2.5.4  <script> の中身。"<" が来てもタグにしない
    ScriptDataLessThanSign, // 13.2.5.15 "<" を読んだ直後
    ScriptDataEndTagOpen,   // 13.2.5.16 "</" を読んだ直後
    ScriptDataEndTagName,   // 13.2.5.17 終了タグ名らしきものの途中
}

enum HtmlToken {
    EndTag { tag: String },
    Char(char),
}

struct HtmlTokenizer {
    chars: Vec<char>,
    pos: usize,
    state: State,
    reconsume: bool,
    last_start_tag: String, // 最後に出した開始タグの名前。同じ名前の終了タグだけが「適切な終了タグ」
    current: String,        // 作りかけの終了タグの名前(小文字に揃える)
    temp: String,           // "</" の後に読んだ文字の控え(原文のまま)。終了タグでなかったら文字として流し直す
    pending: VecDeque<HtmlToken>, // 1 回の遷移で 2 つ以上のトークンを出すときの待ち行列
}

impl HtmlTokenizer {
    /// <script> の直後から読む。本物では <script> を出した時点で状態が ScriptData に切り替わる
    fn new(script_body: &str) -> Self {
        HtmlTokenizer { chars: script_body.chars().collect(), pos: 0, state: State::ScriptData, reconsume: false,
            last_start_tag: "script".to_string(), current: String::new(), temp: String::new(), pending: VecDeque::new() }
    }

    fn consume(&mut self) -> Option<char> {
        if self.reconsume { self.reconsume = false; return self.chars.get(self.pos - 1).copied(); }
        self.pos += 1;
        self.chars.get(self.pos - 1).copied()
    }

    fn reconsume_in(&mut self, state: State) { self.reconsume = true; self.state = state; }

    /// 次のトークン。入力が尽きたら None
    fn next_token(&mut self) -> Option<HtmlToken> {
        if let Some(t) = self.pending.pop_front() { return Some(t); } // 前回の遷移で積み残したトークンを先に出す
        loop {
            let c = self.consume()?;
            match self.state {
                State::ScriptData => match c {
                    '<' => self.state = State::ScriptDataLessThanSign,
                    _ => return Some(HtmlToken::Char(c)),
                },
                State::ScriptDataLessThanSign => match c {
                    '/' => { self.temp.clear(); self.current.clear(); self.state = State::ScriptDataEndTagOpen; }
                    _ => { self.reconsume_in(State::ScriptData); return Some(HtmlToken::Char('<')); } // "<" はただの文字だった
                },
                State::ScriptDataEndTagOpen => match c {
                    c if c.is_ascii_alphabetic() => self.reconsume_in(State::ScriptDataEndTagName),
                    // "</" を 2 文字として出す。1 つ目を返し、2 つ目は待ち行列へ
                    _ => { self.pending.push_back(HtmlToken::Char('/')); self.reconsume_in(State::ScriptData); return Some(HtmlToken::Char('<')); }
                },
                State::ScriptDataEndTagName => match c {
                    '>' if self.current == self.last_start_tag => { self.state = State::ScriptData; return Some(HtmlToken::EndTag { tag: self.current.clone() }); }
                    c if c.is_ascii_alphabetic() => { self.current.push(c.to_ascii_lowercase()); self.temp.push(c); }
                    _ => {
                        // 終了タグではなかった。"</" + temp をそのまま文字として流し、script data に戻る
                        self.pending.push_back(HtmlToken::Char('/'));
                        self.pending.extend(self.temp.chars().map(HtmlToken::Char));
                        self.reconsume_in(State::ScriptData);
                        return Some(HtmlToken::Char('<'));
                    }
                },
            }
        }
    }
}

fn main() {
    for src in ["if (a < b) { x = '</div>'; }</script>", "s = \"</scrip\" + \"t>\";</SCRIPT>"].iter() {
        let mut t = HtmlTokenizer::new(src);
        let mut text = String::new();
        while let Some(token) = t.next_token() {
            match token {
                HtmlToken::Char(c) => text.push(c),
                HtmlToken::EndTag { tag } => println!("文字として読んだ中身: {text:?}\n終了タグ: </{tag}>\n"),
            }
        }
    }
}

use std::collections::VecDeque;

/// {{name}} を置き換える小さなテンプレート。"{" を 1 つ見ただけでは決められないので、
/// 読んだ文字を temp に取っておき、違ったら pending に積んで文字として流し直す(script data と同じ型)
#[derive(Debug, Clone, Copy, PartialEq)]
enum State {
    Text,
    OpenBrace,   // "{" を読んだ直後。次も "{" なら placeholder、違えば "{" は文字
    Placeholder, // "{{" の中
    CloseBrace,  // placeholder の中で "}" を読んだ直後。次も "}" なら終わり、違えば "}" は名前の一部
}

enum Piece {
    Char(char),
    Placeholder(String),
}

fn tokenize(src: &str) -> Vec<Piece> {
    let mut out = Vec::new();
    let mut pending: VecDeque<Piece> = VecDeque::new();
    let mut temp = String::new(); // placeholder になるかもしれない文字の控え
    let mut state = State::Text;
    for c in src.chars() {
        match state {
            State::Text => match c {
                '{' => state = State::OpenBrace,
                _ => out.push(Piece::Char(c)),
            },
            State::OpenBrace => match c {
                '{' => {
                    temp.clear();
                    state = State::Placeholder;
                }
                _ => {
                    // "{" の次が "{" でなかった。"{" と今の文字を 2 つ出して Text に戻る(待ち行列経由)
                    pending.push_back(Piece::Char('{'));
                    pending.push_back(Piece::Char(c));
                    state = State::Text;
                }
            },
            State::Placeholder => match c {
                '}' => state = State::CloseBrace,
                _ => temp.push(c),
            },
            State::CloseBrace => match c {
                '}' => {
                    out.push(Piece::Placeholder(temp.trim().to_string()));
                    state = State::Text;
                }
                _ => {
                    temp.push('}'); // 閉じではなかった。"}" は名前の一部
                    temp.push(c);
                    state = State::Placeholder;
                }
            },
        }
        out.extend(pending.drain(..));
    }
    // EOF: 開いたままの "{{..." は文字として戻す(HTML の EOF 処理と同じ考え方)
    match state {
        State::OpenBrace => out.push(Piece::Char('{')),
        State::Placeholder | State::CloseBrace => {
            let rest = if state == State::CloseBrace { format!("{{{{{temp}}}") } else { format!("{{{{{temp}") };
            out.extend(rest.chars().map(Piece::Char));
        }
        State::Text => {}
    }
    out
}

fn render(src: &str, vars: &[(&str, &str)]) -> String {
    tokenize(src)
        .iter()
        .map(|p| match p {
            Piece::Char(c) => c.to_string(),
            Piece::Placeholder(name) => match vars.iter().find(|(k, _)| k == name) {
                Some((_, v)) => v.to_string(),
                None => format!("<{name} が未定義>"),
            },
        })
        .collect()
}

fn main() {
    let vars = [("name", "kazu"), ("count", "3")];
    let src = "こんにちは {{name}} さん。未読は {{ count }} 件。JSON の {\"a\":1} は {そのまま}。{{nope}} と {{unfinished";
    println!("{src}");
    println!("{}", render(src, &vars));
}