サンプルコードで身につける: CSS のトークン — 先頭の数文字で種類が決まる

// CSS のトークン(CSS Syntax Level 3 §4)。HTML の字句解析のような状態機械ではなく、
// 「先頭の 1〜2 文字を見て種類を決め、必要なだけ読む」形。
// ここでは ident / hash / delim / whitespace と区切り記号だけ。".note" は Delim('.') + Ident("note") になる。
#[derive(Debug, Clone, PartialEq)]
pub enum CssToken {
    Ident(String),
    Hash(String),
    Delim(char),
    Whitespace,
    Colon,
    Semicolon,
    OpenCurly,
    CloseCurly,
}

fn is_name_start(c: char) -> bool {
    c.is_ascii_alphabetic() || c == '_' || !c.is_ascii()
}
fn is_name_char(c: char) -> bool {
    is_name_start(c) || c.is_ascii_digit() || c == '-'
}

/// 仕様 "consume a name": name code point が続く限り読む
fn consume_name(cs: &[char], pos: &mut usize) -> String {
    let start = *pos;
    while *pos < cs.len() && is_name_char(cs[*pos]) {
        *pos += 1;
    }
    cs[start..*pos].iter().collect()
}

pub fn tokenize(css: &str) -> Vec<CssToken> {
    let cs: Vec<char> = css.chars().collect();
    let mut pos = 0;
    let mut out = Vec::new();
    while pos < cs.len() {
        let c = cs[pos];
        if c.is_whitespace() {
            while pos < cs.len() && cs[pos].is_whitespace() {
                pos += 1; // 連続する空白は 1 つの <whitespace-token>
            }
            out.push(CssToken::Whitespace);
        } else if is_name_start(c) {
            out.push(CssToken::Ident(consume_name(&cs, &mut pos)));
        } else if c == '#' && pos + 1 < cs.len() && is_name_char(cs[pos + 1]) {
            pos += 1;
            out.push(CssToken::Hash(consume_name(&cs, &mut pos)));
        } else {
            pos += 1;
            out.push(match c {
                ':' => CssToken::Colon,
                ';' => CssToken::Semicolon,
                '{' => CssToken::OpenCurly,
                '}' => CssToken::CloseCurly,
                other => CssToken::Delim(other), // どれでもない 1 文字
            });
        }
    }
    out
}

fn main() {
    for css in [".note { color: #333; }", "#main", "p > a", "*"] {
        println!("{:?}", css);
        for t in tokenize(css) {
            println!("    {:?}", t);
        }
    }
}

// 数値系 3 種(number / percentage / dimension)と文字列、コメントの消え方。CSS Syntax Level 3 §4.3.3
// "consume a numeric token": 数の直後が "%" なら percentage、識別子が続けば dimension、それ以外は number。
#[derive(Debug, Clone, PartialEq)]
pub enum CssToken { Ident(String), Str(String), Number(f64), Percentage(f64), Dimension(f64, String), Delim(char), Whitespace }

pub struct CssTokenizer {
    chars: Vec<char>,
    pos: usize,
}

impl CssTokenizer {
    pub fn new(css: &str) -> Self {
        CssTokenizer { chars: css.chars().collect(), pos: 0 }
    }
    fn peek(&self, off: usize) -> Option<char> {
        self.chars.get(self.pos + off).copied()
    }
    fn peek_is(&self, off: usize, f: impl Fn(char) -> bool) -> bool {
        self.peek(off).map_or(false, f)
    }
    fn take_while(&mut self, f: impl Fn(char) -> bool) -> String {
        let start = self.pos;
        while self.peek_is(0, &f) {
            self.pos += 1;
        }
        self.chars[start..self.pos].iter().collect()
    }
    /// コメントはトークンにならず、読み捨てる(仕様 "consume comments")
    fn skip_comments(&mut self) {
        while self.peek(0) == Some('/') && self.peek(1) == Some('*') {
            self.pos += 2;
            while self.peek(0).is_some() && !(self.peek(0) == Some('*') && self.peek(1) == Some('/')) {
                self.pos += 1;
            }
            self.pos = (self.pos + 2).min(self.chars.len()); // "*/" を読み捨てる
        }
    }
    fn consume_numeric(&mut self) -> CssToken {
        let mut text = self.take_while(|c| c == '+' || c == '-'); // 符号(あれば)
        text.push_str(&self.take_while(|c| c.is_ascii_digit()));
        if self.peek(0) == Some('.') && self.peek_is(1, |c| c.is_ascii_digit()) {
            self.pos += 1;
            text.push('.');
            text.push_str(&self.take_while(|c| c.is_ascii_digit()));
        }
        let value: f64 = text.parse().unwrap_or(0.0);
        if self.peek(0) == Some('%') {
            self.pos += 1;
            CssToken::Percentage(value)
        } else if self.peek_is(0, |c| c.is_ascii_alphabetic()) {
            CssToken::Dimension(value, self.take_while(|c| c.is_ascii_alphanumeric()))
        } else {
            CssToken::Number(value)
        }
    }
}

impl Iterator for CssTokenizer {
    type Item = CssToken;
    fn next(&mut self) -> Option<CssToken> {
        self.skip_comments();
        let c = self.peek(0)?;
        let starts_number = c.is_ascii_digit() || (matches!(c, '+' | '-' | '.') && self.peek_is(1, |d| d.is_ascii_digit()));
        Some(if c.is_whitespace() {
            self.take_while(|c| c.is_whitespace());
            CssToken::Whitespace
        } else if starts_number {
            self.consume_numeric()
        } else if c.is_ascii_alphabetic() {
            CssToken::Ident(self.take_while(|c| c.is_ascii_alphanumeric() || c == '-'))
        } else if c == '"' || c == '\'' {
            self.pos += 1;
            let s = self.take_while(|d| d != c);
            self.pos = (self.pos + 1).min(self.chars.len()); // 閉じ引用符
            CssToken::Str(s)
        } else {
            self.pos += 1;
            CssToken::Delim(c)
        })
    }
}

fn main() {
    for css in ["16px", "1.5em", "50%", "-2", ".5", "\"Noto Sans\"", "a /* コメント */ b", "0 auto"] {
        let toks: Vec<CssToken> = CssTokenizer::new(css).collect();
        println!("{:<20} -> {:?}", css, toks);
    }
}

// 実務に写す: 設定ファイルの "512MB" "50%" "1.5s" "8" のような値を、CSS の number / percentage /
// dimension と同じ 3 種類に読み分ける。「数を読み、直後に % があれば割合、英字が続けば単位つき、
// 何も無ければ素の数」という判定は CSS Syntax の consume a numeric token そのもの。
#[derive(Debug, Clone, PartialEq)]
pub enum Quantity {
    Number(f64),
    Percentage(f64),
    Dimension(f64, String),
}

#[derive(Debug, PartialEq)]
pub enum ParseError {
    Empty,
    NotANumber(String),
    TrailingGarbage(String),
}

pub fn parse_quantity(text: &str) -> Result<Quantity, ParseError> {
    let s = text.trim();
    if s.is_empty() {
        return Err(ParseError::Empty);
    }
    // 数の部分: 符号・数字・小数点が続く限り
    let num_len = s
        .char_indices()
        .take_while(|&(i, c)| c.is_ascii_digit() || c == '.' || (i == 0 && (c == '-' || c == '+')))
        .count();
    let (num, rest) = s.split_at(num_len);
    let value: f64 = num.parse().map_err(|_| ParseError::NotANumber(s.to_string()))?;
    if rest.is_empty() {
        Ok(Quantity::Number(value))
    } else if rest == "%" {
        Ok(Quantity::Percentage(value))
    } else if rest.chars().all(|c| c.is_ascii_alphabetic()) {
        Ok(Quantity::Dimension(value, rest.to_string()))
    } else {
        Err(ParseError::TrailingGarbage(rest.to_string()))
    }
}

/// 単位つきの量をバイト数や秒数に直す(単位表は用途ごとに決める)
pub fn to_bytes(q: &Quantity, total: f64) -> Option<f64> {
    match q {
        Quantity::Number(n) => Some(*n),
        Quantity::Percentage(p) => Some(total * p / 100.0),
        Quantity::Dimension(n, unit) => match unit.as_str() {
            "KB" => Some(n * 1024.0),
            "MB" => Some(n * 1024.0 * 1024.0),
            "GB" => Some(n * 1024.0 * 1024.0 * 1024.0),
            _ => None,
        },
    }
}

fn main() {
    let total_memory = 8.0 * 1024.0 * 1024.0 * 1024.0; // 8GB
    for text in ["512MB", "50%", "8", "1.5GB", "2s", "", "abc", "10px!"] {
        match parse_quantity(text) {
            Ok(q) => {
                let bytes = to_bytes(&q, total_memory).map(|b| format!("{:.0} bytes", b)).unwrap_or("(メモリの単位でない)".into());
                println!("{:<8} -> {:<28} {}", format!("{:?}", text), format!("{:?}", q), bytes);
            }
            Err(e) => println!("{:<8} -> Err({:?})", format!("{:?}", text), e),
        }
    }
}