123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212 |
- """
- pygments.token
- ~~~~~~~~~~~~~~
- Basic token types and the standard tokens.
- :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
- :license: BSD, see LICENSE for details.
- """
- class _TokenType(tuple):
- parent = None
- def split(self):
- buf = []
- node = self
- while node is not None:
- buf.append(node)
- node = node.parent
- buf.reverse()
- return buf
- def __init__(self, *args):
- # no need to call super.__init__
- self.subtypes = set()
- def __contains__(self, val):
- return self is val or (
- type(val) is self.__class__ and
- val[:len(self)] == self
- )
- def __getattr__(self, val):
- if not val or not val[0].isupper():
- return tuple.__getattribute__(self, val)
- new = _TokenType(self + (val,))
- setattr(self, val, new)
- self.subtypes.add(new)
- new.parent = self
- return new
- def __repr__(self):
- return 'Token' + (self and '.' or '') + '.'.join(self)
- def __copy__(self):
- # These instances are supposed to be singletons
- return self
- def __deepcopy__(self, memo):
- # These instances are supposed to be singletons
- return self
- Token = _TokenType()
- # Special token types
- Text = Token.Text
- Whitespace = Text.Whitespace
- Escape = Token.Escape
- Error = Token.Error
- # Text that doesn't belong to this lexer (e.g. HTML in PHP)
- Other = Token.Other
- # Common token types for source code
- Keyword = Token.Keyword
- Name = Token.Name
- Literal = Token.Literal
- String = Literal.String
- Number = Literal.Number
- Punctuation = Token.Punctuation
- Operator = Token.Operator
- Comment = Token.Comment
- # Generic types for non-source code
- Generic = Token.Generic
- # String and some others are not direct children of Token.
- # alias them:
- Token.Token = Token
- Token.String = String
- Token.Number = Number
- def is_token_subtype(ttype, other):
- """
- Return True if ``ttype`` is a subtype of ``other``.
- exists for backwards compatibility. use ``ttype in other`` now.
- """
- return ttype in other
- def string_to_tokentype(s):
- """
- Convert a string into a token type::
- >>> string_to_token('String.Double')
- Token.Literal.String.Double
- >>> string_to_token('Token.Literal.Number')
- Token.Literal.Number
- >>> string_to_token('')
- Token
- Tokens that are already tokens are returned unchanged:
- >>> string_to_token(String)
- Token.Literal.String
- """
- if isinstance(s, _TokenType):
- return s
- if not s:
- return Token
- node = Token
- for item in s.split('.'):
- node = getattr(node, item)
- return node
- # Map standard token types to short names, used in CSS class naming.
- # If you add a new item, please be sure to run this file to perform
- # a consistency check for duplicate values.
- STANDARD_TYPES = {
- Token: '',
- Text: '',
- Whitespace: 'w',
- Escape: 'esc',
- Error: 'err',
- Other: 'x',
- Keyword: 'k',
- Keyword.Constant: 'kc',
- Keyword.Declaration: 'kd',
- Keyword.Namespace: 'kn',
- Keyword.Pseudo: 'kp',
- Keyword.Reserved: 'kr',
- Keyword.Type: 'kt',
- Name: 'n',
- Name.Attribute: 'na',
- Name.Builtin: 'nb',
- Name.Builtin.Pseudo: 'bp',
- Name.Class: 'nc',
- Name.Constant: 'no',
- Name.Decorator: 'nd',
- Name.Entity: 'ni',
- Name.Exception: 'ne',
- Name.Function: 'nf',
- Name.Function.Magic: 'fm',
- Name.Property: 'py',
- Name.Label: 'nl',
- Name.Namespace: 'nn',
- Name.Other: 'nx',
- Name.Tag: 'nt',
- Name.Variable: 'nv',
- Name.Variable.Class: 'vc',
- Name.Variable.Global: 'vg',
- Name.Variable.Instance: 'vi',
- Name.Variable.Magic: 'vm',
- Literal: 'l',
- Literal.Date: 'ld',
- String: 's',
- String.Affix: 'sa',
- String.Backtick: 'sb',
- String.Char: 'sc',
- String.Delimiter: 'dl',
- String.Doc: 'sd',
- String.Double: 's2',
- String.Escape: 'se',
- String.Heredoc: 'sh',
- String.Interpol: 'si',
- String.Other: 'sx',
- String.Regex: 'sr',
- String.Single: 's1',
- String.Symbol: 'ss',
- Number: 'm',
- Number.Bin: 'mb',
- Number.Float: 'mf',
- Number.Hex: 'mh',
- Number.Integer: 'mi',
- Number.Integer.Long: 'il',
- Number.Oct: 'mo',
- Operator: 'o',
- Operator.Word: 'ow',
- Punctuation: 'p',
- Comment: 'c',
- Comment.Hashbang: 'ch',
- Comment.Multiline: 'cm',
- Comment.Preproc: 'cp',
- Comment.PreprocFile: 'cpf',
- Comment.Single: 'c1',
- Comment.Special: 'cs',
- Generic: 'g',
- Generic.Deleted: 'gd',
- Generic.Emph: 'ge',
- Generic.Error: 'gr',
- Generic.Heading: 'gh',
- Generic.Inserted: 'gi',
- Generic.Output: 'go',
- Generic.Prompt: 'gp',
- Generic.Strong: 'gs',
- Generic.Subheading: 'gu',
- Generic.Traceback: 'gt',
- }
|