サンプルコードで身につける: 状態機械という読み方 — 1 文字読んで、状態を変えるか、トークンを出すか

/// 状態。名前は HTML Standard 13.2.5 の節の名前に合わせる
#[derive(Debug, Clone, Copy, PartialEq)]
enum State {
    Data,       // タグの外側
    TagOpen,    // "<" を読んだ直後
    EndTagOpen, // "</" を読んだ直後
    TagName,    // タグ名の途中
}

/// 字句解析の出力。この回は属性を省く
enum HtmlToken {
    StartTag { tag: String },
    EndTag { tag: String },
    Char(char),
    Eof,
}

fn tokenize(html: &str) -> Vec<HtmlToken> {
    let mut tokens = Vec::new();
    let mut state = State::Data;
    let mut name = String::new(); // 作りかけのタグ名
    let mut is_end = false;
    for c in html.chars() {
        // 「今の状態」と「今の文字」の 2 つで、次にすることが決まる
        match state {
            State::Data => match c {
                '<' => state = State::TagOpen,
                _ => tokens.push(HtmlToken::Char(c)),
            },
            State::TagOpen => match c {
                '/' => state = State::EndTagOpen,
                c if c.is_ascii_alphabetic() => {
                    name.clear();
                    is_end = false;
                    name.push(c.to_ascii_lowercase());
                    state = State::TagName;
                }
                _ => {
                    // "<" の次がタグ名でなければ "<" はただの文字(例: a < b)
                    tokens.push(HtmlToken::Char('<'));
                    tokens.push(HtmlToken::Char(c));
                    state = State::Data;
                }
            },
            State::EndTagOpen => {
                name.clear();
                is_end = true;
                name.push(c.to_ascii_lowercase());
                state = State::TagName;
            }
            State::TagName => match c {
                '>' => {
                    let tag = name.clone();
                    tokens.push(if is_end { HtmlToken::EndTag { tag } } else { HtmlToken::StartTag { tag } });
                    state = State::Data;
                }
                _ => name.push(c.to_ascii_lowercase()),
            },
        }
    }
    tokens.push(HtmlToken::Eof); // 入力が尽きたら、どの状態でも EOF トークンを 1 つ出す
    tokens
}

fn main() {
    for t in tokenize("<P>a < b<BR></p>") {
        match t {
            HtmlToken::StartTag { tag } => println!("StartTag <{tag}>"),
            HtmlToken::EndTag { tag } => println!("EndTag   </{tag}>"),
            HtmlToken::Char(c) => println!("Char     {c:?}"),
            HtmlToken::Eof => println!("Eof"),
        }
    }
}

#[derive(Debug, Clone, Copy, PartialEq)]
enum State {
    Data,
    TagOpen,
    EndTagOpen,
    TagName,
}

enum HtmlToken {
    StartTag { tag: String },
    EndTag { tag: String },
    Char(char),
    Eof,
}

struct HtmlTokenizer {
    chars: Vec<char>,
    pos: usize,
    state: State,
    /// true なら次の consume() は直前の文字をもう一度返す(仕様の "reconsume")
    reconsume: bool,
    current: String, // 作りかけのタグ名
    is_end: bool,    // 作りかけのタグが終了タグか
}

impl HtmlTokenizer {
    fn consume(&mut self) -> Option<char> {
        if self.reconsume {
            self.reconsume = false;
            return self.chars.get(self.pos - 1).copied();
        }
        self.pos += 1;
        self.chars.get(self.pos - 1).copied()
    }

    /// 「state に切り替えて、今の文字をもう一度読む」。仕様の "Reconsume in the X state" そのもの
    fn reconsume_in(&mut self, state: State) {
        self.reconsume = true;
        self.state = state;
    }

    /// 1 トークン進める。読んだ文字と状態を 1 行ずつ出す(学習用のトレース)
    fn next_token(&mut self) -> HtmlToken {
        loop {
            let again = self.reconsume;
            let Some(c) = self.consume() else { return HtmlToken::Eof }; // 入力が尽きたら EOF
            println!("  {:<11} {:?}{}", format!("{:?}", self.state), c, if again { "  <- 同じ文字をもう一度" } else { "" });
            match self.state {
                State::Data => match c {
                    '<' => self.state = State::TagOpen,
                    _ => return HtmlToken::Char(c),
                },
                State::TagOpen => match c {
                    '/' => self.state = State::EndTagOpen,
                    c if c.is_ascii_alphabetic() => {
                        self.current.clear();
                        self.is_end = false;
                        self.reconsume_in(State::TagName); // 文字を捨てずに TagName に読み直させる
                    }
                    _ => {
                        self.reconsume_in(State::Data); // "<" は文字だった。今の文字は Data で読み直す
                        return HtmlToken::Char('<');
                    }
                },
                State::EndTagOpen => { self.current.clear(); self.is_end = true; self.reconsume_in(State::TagName); }
                State::TagName => match c {
                    '>' => {
                        self.state = State::Data;
                        let tag = self.current.clone();
                        return if self.is_end { HtmlToken::EndTag { tag } } else { HtmlToken::StartTag { tag } };
                    }
                    _ => self.current.push(c.to_ascii_lowercase()),
                },
            }
        }
    }
}

fn main() {
    let mut t = HtmlTokenizer { chars: "a <<p>x</p>".chars().collect(), pos: 0, state: State::Data, reconsume: false, current: String::new(), is_end: false };
    loop {
        match t.next_token() {
            HtmlToken::Char(c) => println!("=> Char {c:?}"),
            HtmlToken::StartTag { tag } => println!("=> StartTag <{tag}>"),
            HtmlToken::EndTag { tag } => println!("=> EndTag </{tag}>"),
            HtmlToken::Eof => return println!("=> Eof"),
        }
    }
}

/// CSV(RFC 4180 の形)を読む状態機械。HTML の字句解析と同じ「状態 x 文字 -> 次の動き」で書ける
#[derive(Debug, Clone, Copy, PartialEq)]
enum State {
    FieldStart, // フィールドの先頭。引用符で始まるか、そうでないかが分かれる
    Unquoted,   // 引用符なしのフィールドの途中
    Quoted,     // "..." の中。カンマや改行もただの文字
    AfterQuote, // Quoted の中で '"' を読んだ直後。次が '"' なら '"' の文字、そうでなければ閉じ
}

fn parse_csv(text: &str) -> Vec<Vec<String>> {
    let chars: Vec<char> = text.chars().collect();
    let mut records = Vec::new();
    let mut record: Vec<String> = Vec::new();
    let mut field = String::new();
    let mut state = State::FieldStart;
    let mut i = 0;
    while i < chars.len() {
        let c = chars[i];
        match state {
            State::FieldStart => match c {
                '"' => state = State::Quoted,
                ',' => record.push(std::mem::take(&mut field)),
                '\n' => {
                    record.push(std::mem::take(&mut field));
                    records.push(std::mem::take(&mut record));
                }
                _ => {
                    field.push(c);
                    state = State::Unquoted;
                }
            },
            State::Unquoted => match c {
                ',' => {
                    record.push(std::mem::take(&mut field));
                    state = State::FieldStart;
                }
                '\n' => {
                    record.push(std::mem::take(&mut field));
                    records.push(std::mem::take(&mut record));
                    state = State::FieldStart;
                }
                _ => field.push(c),
            },
            State::Quoted => match c {
                '"' => state = State::AfterQuote,
                _ => field.push(c),
            },
            State::AfterQuote => match c {
                '"' => {
                    field.push('"'); // "" は 1 つの '"'
                    state = State::Quoted;
                }
                _ => {
                    // 閉じ引用符だった。今の文字(',' や '\n')は Unquoted の規則で読み直す = reconsume
                    state = State::Unquoted;
                    continue; // i を進めない
                }
            },
        }
        i += 1;
    }
    // 入力の終わり(EOF)。作りかけのフィールドがあれば確定する
    if !field.is_empty() || !record.is_empty() {
        record.push(field);
        records.push(record);
    }
    records
}

fn main() {
    let csv = "name,comment,price\n\"Yamada, Taro\",\"He said \"\"hi\"\"\",1200\nSuzuki,\"two\nlines\",800";
    for (i, record) in parse_csv(csv).iter().enumerate() {
        println!("record {i}: {record:?}");
    }
}