token.py 6.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212
  1. """
  2. pygments.token
  3. ~~~~~~~~~~~~~~
  4. Basic token types and the standard tokens.
  5. :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
  6. :license: BSD, see LICENSE for details.
  7. """
  8. class _TokenType(tuple):
  9. parent = None
  10. def split(self):
  11. buf = []
  12. node = self
  13. while node is not None:
  14. buf.append(node)
  15. node = node.parent
  16. buf.reverse()
  17. return buf
  18. def __init__(self, *args):
  19. # no need to call super.__init__
  20. self.subtypes = set()
  21. def __contains__(self, val):
  22. return self is val or (
  23. type(val) is self.__class__ and
  24. val[:len(self)] == self
  25. )
  26. def __getattr__(self, val):
  27. if not val or not val[0].isupper():
  28. return tuple.__getattribute__(self, val)
  29. new = _TokenType(self + (val,))
  30. setattr(self, val, new)
  31. self.subtypes.add(new)
  32. new.parent = self
  33. return new
  34. def __repr__(self):
  35. return 'Token' + (self and '.' or '') + '.'.join(self)
  36. def __copy__(self):
  37. # These instances are supposed to be singletons
  38. return self
  39. def __deepcopy__(self, memo):
  40. # These instances are supposed to be singletons
  41. return self
  42. Token = _TokenType()
  43. # Special token types
  44. Text = Token.Text
  45. Whitespace = Text.Whitespace
  46. Escape = Token.Escape
  47. Error = Token.Error
  48. # Text that doesn't belong to this lexer (e.g. HTML in PHP)
  49. Other = Token.Other
  50. # Common token types for source code
  51. Keyword = Token.Keyword
  52. Name = Token.Name
  53. Literal = Token.Literal
  54. String = Literal.String
  55. Number = Literal.Number
  56. Punctuation = Token.Punctuation
  57. Operator = Token.Operator
  58. Comment = Token.Comment
  59. # Generic types for non-source code
  60. Generic = Token.Generic
  61. # String and some others are not direct children of Token.
  62. # alias them:
  63. Token.Token = Token
  64. Token.String = String
  65. Token.Number = Number
  66. def is_token_subtype(ttype, other):
  67. """
  68. Return True if ``ttype`` is a subtype of ``other``.
  69. exists for backwards compatibility. use ``ttype in other`` now.
  70. """
  71. return ttype in other
  72. def string_to_tokentype(s):
  73. """
  74. Convert a string into a token type::
  75. >>> string_to_token('String.Double')
  76. Token.Literal.String.Double
  77. >>> string_to_token('Token.Literal.Number')
  78. Token.Literal.Number
  79. >>> string_to_token('')
  80. Token
  81. Tokens that are already tokens are returned unchanged:
  82. >>> string_to_token(String)
  83. Token.Literal.String
  84. """
  85. if isinstance(s, _TokenType):
  86. return s
  87. if not s:
  88. return Token
  89. node = Token
  90. for item in s.split('.'):
  91. node = getattr(node, item)
  92. return node
  93. # Map standard token types to short names, used in CSS class naming.
  94. # If you add a new item, please be sure to run this file to perform
  95. # a consistency check for duplicate values.
  96. STANDARD_TYPES = {
  97. Token: '',
  98. Text: '',
  99. Whitespace: 'w',
  100. Escape: 'esc',
  101. Error: 'err',
  102. Other: 'x',
  103. Keyword: 'k',
  104. Keyword.Constant: 'kc',
  105. Keyword.Declaration: 'kd',
  106. Keyword.Namespace: 'kn',
  107. Keyword.Pseudo: 'kp',
  108. Keyword.Reserved: 'kr',
  109. Keyword.Type: 'kt',
  110. Name: 'n',
  111. Name.Attribute: 'na',
  112. Name.Builtin: 'nb',
  113. Name.Builtin.Pseudo: 'bp',
  114. Name.Class: 'nc',
  115. Name.Constant: 'no',
  116. Name.Decorator: 'nd',
  117. Name.Entity: 'ni',
  118. Name.Exception: 'ne',
  119. Name.Function: 'nf',
  120. Name.Function.Magic: 'fm',
  121. Name.Property: 'py',
  122. Name.Label: 'nl',
  123. Name.Namespace: 'nn',
  124. Name.Other: 'nx',
  125. Name.Tag: 'nt',
  126. Name.Variable: 'nv',
  127. Name.Variable.Class: 'vc',
  128. Name.Variable.Global: 'vg',
  129. Name.Variable.Instance: 'vi',
  130. Name.Variable.Magic: 'vm',
  131. Literal: 'l',
  132. Literal.Date: 'ld',
  133. String: 's',
  134. String.Affix: 'sa',
  135. String.Backtick: 'sb',
  136. String.Char: 'sc',
  137. String.Delimiter: 'dl',
  138. String.Doc: 'sd',
  139. String.Double: 's2',
  140. String.Escape: 'se',
  141. String.Heredoc: 'sh',
  142. String.Interpol: 'si',
  143. String.Other: 'sx',
  144. String.Regex: 'sr',
  145. String.Single: 's1',
  146. String.Symbol: 'ss',
  147. Number: 'm',
  148. Number.Bin: 'mb',
  149. Number.Float: 'mf',
  150. Number.Hex: 'mh',
  151. Number.Integer: 'mi',
  152. Number.Integer.Long: 'il',
  153. Number.Oct: 'mo',
  154. Operator: 'o',
  155. Operator.Word: 'ow',
  156. Punctuation: 'p',
  157. Comment: 'c',
  158. Comment.Hashbang: 'ch',
  159. Comment.Multiline: 'cm',
  160. Comment.Preproc: 'cp',
  161. Comment.PreprocFile: 'cpf',
  162. Comment.Single: 'c1',
  163. Comment.Special: 'cs',
  164. Generic: 'g',
  165. Generic.Deleted: 'gd',
  166. Generic.Emph: 'ge',
  167. Generic.Error: 'gr',
  168. Generic.Heading: 'gh',
  169. Generic.Inserted: 'gi',
  170. Generic.Output: 'go',
  171. Generic.Prompt: 'gp',
  172. Generic.Strong: 'gs',
  173. Generic.Subheading: 'gu',
  174. Generic.Traceback: 'gt',
  175. }