spdx/

lexer.rs

use crate::{
    error::{ParseError, Reason},
    ExceptionId, LicenseId,
};

/// Parsing configuration for SPDX expression
#[derive(Default, Copy, Clone)]
pub struct ParseMode {
    /// The `AND`, `OR`, and `WITH` operators are required to be uppercase in
    /// the SPDX spec, but enabling this option allows them to be lowercased
    pub allow_lower_case_operators: bool,
    /// Allows the use of `/` as a synonym for the `OR` operator.
    ///
    /// This also allows for not having whitespace between the `/` and the terms
    /// on either side
    pub allow_slash_as_or_operator: bool,
    /// Allows some invalid/imprecise identifiers as synonyms for an actual
    /// license identifier.
    ///
    /// See [`IMPRECISE_NAMES`](crate::identifiers::IMPRECISE_NAMES) for a list
    /// of the current synonyms. Note that this list is not comprehensive but
    /// can be expanded upon when invalid identifiers are found in the wild.
    pub allow_imprecise_license_names: bool,
    /// The various GPL licenses diverge from every other license in the SPDX
    /// license list by having an `-or-later` variant that is used as a suffix
    /// on a base license (eg. `GPL-3.0-or-later`) rather than the canonical
    /// `GPL-3.0+`.
    ///
    /// This option just allows GPL licenses to be treated similarly to all of
    /// the other SPDX licenses.
    pub allow_postfix_plus_on_gpl: bool,
}

impl ParseMode {
    /// Strict, specification compliant SPDX parsing.
    ///
    /// 1. Only license identifiers in the SPDX license list, or
    ///     Document/LicenseRef, are allowed. The license identifiers are also
    ///     case-sensitive.
    /// 1. `WITH`, `AND`, and `OR` are the only valid operators
    pub const STRICT: Self = Self {
        allow_lower_case_operators: false,
        allow_slash_as_or_operator: false,
        allow_imprecise_license_names: false,
        allow_postfix_plus_on_gpl: false,
    };

    /// Allow non-conforming syntax for crates-io compatibility
    ///
    /// 1. Additional, invalid, identifiers are accepted and mapped to a correct
    ///     SPDX license identifier.
    ///     See [`IMPRECISE_NAMES`](crate::identifiers::IMPRECISE_NAMES) for the
    ///     list of additionally accepted identifiers and the license they
    ///     correspond to.
    /// 1. `/` can by used as a synonym for `OR`, and doesn't need to be
    ///     separated by whitespace from the terms it combines
    pub const LAX: Self = Self {
        allow_lower_case_operators: true,
        allow_slash_as_or_operator: true,
        allow_imprecise_license_names: true,
        allow_postfix_plus_on_gpl: true,
    };
}

/// A single token in an SPDX license expression
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Token<'a> {
    /// A recognized SPDX license id
    Spdx(LicenseId),
    /// A `LicenseRef-` prefixed id, with an optional `DocumentRef-`
    LicenseRef {
        doc_ref: Option<&'a str>,
        lic_ref: &'a str,
    },
    /// A recognized SPDX exception id
    Exception(ExceptionId),
    /// A postfix `+` indicating "or later" for a particular SPDX license id
    Plus,
    /// A `(` for starting a group
    OpenParen,
    /// A `)` for ending a group
    CloseParen,
    /// A `WITH` operator
    With,
    /// An `AND` operator
    And,
    /// An `OR` operator
    Or,
}

impl<'a> std::fmt::Display for Token<'a> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        std::fmt::Debug::fmt(self, f)
    }
}

impl<'a> Token<'a> {
    fn len(&self) -> usize {
        match self {
            Token::Spdx(id) => id.name.len(),
            Token::Exception(e) => e.name.len(),
            Token::With => 4,
            Token::And => 3,
            Token::Or => 2,
            Token::Plus | Token::OpenParen | Token::CloseParen => 1,
            Token::LicenseRef { doc_ref, lic_ref } => {
                doc_ref.map_or(0, |d| {
                    // +1 is for the `:`
                    "DocumentRef-".len() + d.len() + 1
                }) + "LicenseRef-".len()
                    + lic_ref.len()
            }
        }
    }
}

/// Allows iteration through an SPDX license expression, yielding
/// a token or a `ParseError`.
///
/// Prefer to use `Expression::parse` or `Licensee::parse` rather
/// than directly using the lexer
pub struct Lexer<'a> {
    inner: &'a str,
    original: &'a str,
    offset: usize,
    mode: ParseMode,
}

impl<'a> Lexer<'a> {
    /// Creates a Lexer over a license expression
    #[must_use]
    pub fn new(text: &'a str) -> Self {
        Self {
            inner: text,
            original: text,
            offset: 0,
            mode: ParseMode::STRICT,
        }
    }

    /// Creates a Lexer over a license expression
    ///
    /// With `ParseMode::Lax` it allows non-conforming syntax
    /// used in crates-io crates.
    #[must_use]
    pub fn new_mode(text: &'a str, mode: ParseMode) -> Self {
        Self {
            inner: text,
            original: text,
            offset: 0,
            mode,
        }
    }

    #[inline]
    fn is_ref_char(c: &char) -> bool {
        c.is_ascii_alphanumeric() || *c == '-' || *c == '.'
    }

    /// Return a matching text token if found - equivalent to the regex `^[-a-zA-Z0-9.:]+`
    fn find_text_token(text: &'a str) -> Option<&'a str> {
        let is_token_char = |c: &char| Self::is_ref_char(c) || *c == ':';
        match text.chars().take_while(is_token_char).count() {
            index if index > 0 => Some(&text[..index]),
            _ => None,
        }
    }

    /// Extract the text after `prefix` if made up of valid ref characters
    fn find_ref(prefix: &str, text: &'a str) -> Option<&'a str> {
        text.strip_prefix(prefix).map(|value| {
            let end = value.chars().take_while(Self::is_ref_char).count();
            &text[prefix.len()..prefix.len() + end]
        })
    }

    /// Return a license ref if found - equivalent to the regex `^LicenseRef-([-a-zA-Z0-9.]+)`
    #[inline]
    fn find_license_ref(text: &'a str) -> Option<&'a str> {
        Self::find_ref("LicenseRef-", text)
    }

    /// Return a document ref and license ref if found,
    /// equivalent to the regex `^DocumentRef-([-a-zA-Z0-9.]+):LicenseRef-([-a-zA-Z0-9.]+)`
    fn find_document_and_license_ref(text: &'a str) -> Option<(&'a str, &'a str)> {
        let split = text.split_once(':');
        let doc_ref = split.and_then(|(doc, _)| Self::find_ref("DocumentRef-", doc));
        let lic_ref = split.and_then(|(_, lic)| Self::find_license_ref(lic));
        Option::zip(doc_ref, lic_ref)
    }
}

/// A wrapper around a particular token that includes the span of the characters
/// in the original string, for diagnostic purposes
#[derive(Debug)]
pub struct LexerToken<'a> {
    /// The token that was lexed
    pub token: Token<'a>,
    /// The range of the token characters in the original license expression
    pub span: std::ops::Range<usize>,
}

impl<'a> Iterator for Lexer<'a> {
    type Item = Result<LexerToken<'a>, ParseError>;

    fn next(&mut self) -> Option<Self::Item> {
        #[allow(clippy::unnecessary_wraps)]
        fn ok_token(token: Token<'_>) -> Option<Result<(Token<'_>, usize), ParseError>> {
            let len = token.len();
            Some(Ok((token, len)))
        }

        // Jump over any whitespace, updating `self.inner` and `self.offset` appropriately
        let non_whitespace_index = match self.inner.find(|c: char| !c.is_whitespace()) {
            Some(idx) => idx,
            None => self.inner.len(),
        };
        self.inner = &self.inner[non_whitespace_index..];
        self.offset += non_whitespace_index;

        match self.inner.chars().next() {
            None => None,
            // From SPDX 2.1 spec
            // There MUST NOT be whitespace between a license-id and any following "+".
            Some('+') => {
                if non_whitespace_index == 0 {
                    ok_token(Token::Plus)
                } else {
                    Some(Err(ParseError {
                        original: self.original.to_owned(),
                        span: self.offset - non_whitespace_index..self.offset,
                        reason: Reason::SeparatedPlus,
                    }))
                }
            }
            Some('(') => ok_token(Token::OpenParen),
            Some(')') => ok_token(Token::CloseParen),
            Some('/') if self.mode.allow_slash_as_or_operator => Some(Ok((Token::Or, 1))),
            Some(_) => match Lexer::find_text_token(self.inner) {
                None => Some(Err(ParseError {
                    original: self.original.to_owned(),
                    span: self.offset..self.offset + self.inner.len(),
                    reason: Reason::InvalidCharacters,
                })),
                Some(m) => {
                    if m == "WITH" {
                        ok_token(Token::With)
                    } else if m == "AND" {
                        ok_token(Token::And)
                    } else if m == "OR" {
                        ok_token(Token::Or)
                    } else if self.mode.allow_lower_case_operators && m == "and" {
                        ok_token(Token::And)
                    } else if self.mode.allow_lower_case_operators && m == "or" {
                        ok_token(Token::Or)
                    } else if self.mode.allow_lower_case_operators && m == "with" {
                        ok_token(Token::With)
                    } else if let Some(lic_id) = crate::license_id(m) {
                        ok_token(Token::Spdx(lic_id))
                    } else if let Some(exc_id) = crate::exception_id(m) {
                        ok_token(Token::Exception(exc_id))
                    } else if let Some((doc_ref, lic_ref)) = Lexer::find_document_and_license_ref(m)
                    {
                        ok_token(Token::LicenseRef {
                            doc_ref: Some(doc_ref),
                            lic_ref,
                        })
                    } else if let Some(lic_ref) = Lexer::find_license_ref(m) {
                        ok_token(Token::LicenseRef {
                            doc_ref: None,
                            lic_ref,
                        })
                    } else if let Some((lic_id, token_len)) =
                        if self.mode.allow_imprecise_license_names {
                            crate::imprecise_license_id(self.inner)
                        } else {
                            None
                        }
                    {
                        Some(Ok((Token::Spdx(lic_id), token_len)))
                    } else {
                        Some(Err(ParseError {
                            original: self.original.to_owned(),
                            span: self.offset..self.offset + m.len(),
                            reason: Reason::UnknownTerm,
                        }))
                    }
                }
            },
        }
        .map(|res| {
            res.map(|(tok, len)| {
                let start = self.offset;
                self.inner = &self.inner[len..];
                self.offset += len;

                LexerToken {
                    token: tok,
                    span: start..self.offset,
                }
            })
        })
    }
}