You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
214 lines
6.0 KiB
Python
214 lines
6.0 KiB
Python
2 years ago
|
"""
|
||
|
|
pygments.token
|
||
|
|
~~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
Basic token types and the standard tokens.
|
||
|
|
|
||
|
|
:copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
|
||
|
|
:license: BSD, see LICENSE for details.
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
class _TokenType(tuple):
|
||
|
|
parent = None
|
||
|
|
|
||
|
|
def split(self):
|
||
|
|
buf = []
|
||
|
|
node = self
|
||
|
|
while node is not None:
|
||
|
|
buf.append(node)
|
||
|
|
node = node.parent
|
||
|
|
buf.reverse()
|
||
|
|
return buf
|
||
|
|
|
||
|
|
def __init__(self, *args):
|
||
|
|
# no need to call super.__init__
|
||
|
|
self.subtypes = set()
|
||
|
|
|
||
|
|
def __contains__(self, val):
|
||
|
|
return self is val or (
|
||
|
|
type(val) is self.__class__ and
|
||
|
|
val[:len(self)] == self
|
||
|
|
)
|
||
|
|
|
||
|
|
def __getattr__(self, val):
|
||
|
|
if not val or not val[0].isupper():
|
||
|
|
return tuple.__getattribute__(self, val)
|
||
|
|
new = _TokenType(self + (val,))
|
||
|
|
setattr(self, val, new)
|
||
|
|
self.subtypes.add(new)
|
||
|
|
new.parent = self
|
||
|
|
return new
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return 'Token' + (self and '.' or '') + '.'.join(self)
|
||
|
|
|
||
|
|
def __copy__(self):
|
||
|
|
# These instances are supposed to be singletons
|
||
|
|
return self
|
||
|
|
|
||
|
|
def __deepcopy__(self, memo):
|
||
|
|
# These instances are supposed to be singletons
|
||
|
|
return self
|
||
|
|
|
||
|
|
|
||
|
|
Token = _TokenType()
|
||
|
|
|
||
|
|
# Special token types
|
||
|
|
Text = Token.Text
|
||
|
|
Whitespace = Text.Whitespace
|
||
|
|
Escape = Token.Escape
|
||
|
|
Error = Token.Error
|
||
|
|
# Text that doesn't belong to this lexer (e.g. HTML in PHP)
|
||
|
|
Other = Token.Other
|
||
|
|
|
||
|
|
# Common token types for source code
|
||
|
|
Keyword = Token.Keyword
|
||
|
|
Name = Token.Name
|
||
|
|
Literal = Token.Literal
|
||
|
|
String = Literal.String
|
||
|
|
Number = Literal.Number
|
||
|
|
Punctuation = Token.Punctuation
|
||
|
|
Operator = Token.Operator
|
||
|
|
Comment = Token.Comment
|
||
|
|
|
||
|
|
# Generic types for non-source code
|
||
|
|
Generic = Token.Generic
|
||
|
|
|
||
|
|
# String and some others are not direct children of Token.
|
||
|
|
# alias them:
|
||
|
|
Token.Token = Token
|
||
|
|
Token.String = String
|
||
|
|
Token.Number = Number
|
||
|
|
|
||
|
|
|
||
|
|
def is_token_subtype(ttype, other):
|
||
|
|
"""
|
||
|
|
Return True if ``ttype`` is a subtype of ``other``.
|
||
|
|
|
||
|
|
exists for backwards compatibility. use ``ttype in other`` now.
|
||
|
|
"""
|
||
|
|
return ttype in other
|
||
|
|
|
||
|
|
|
||
|
|
def string_to_tokentype(s):
|
||
|
|
"""
|
||
|
|
Convert a string into a token type::
|
||
|
|
|
||
|
|
>>> string_to_token('String.Double')
|
||
|
|
Token.Literal.String.Double
|
||
|
|
>>> string_to_token('Token.Literal.Number')
|
||
|
|
Token.Literal.Number
|
||
|
|
>>> string_to_token('')
|
||
|
|
Token
|
||
|
|
|
||
|
|
Tokens that are already tokens are returned unchanged:
|
||
|
|
|
||
|
|
>>> string_to_token(String)
|
||
|
|
Token.Literal.String
|
||
|
|
"""
|
||
|
|
if isinstance(s, _TokenType):
|
||
|
|
return s
|
||
|
|
if not s:
|
||
|
|
return Token
|
||
|
|
node = Token
|
||
|
|
for item in s.split('.'):
|
||
|
|
node = getattr(node, item)
|
||
|
|
return node
|
||
|
|
|
||
|
|
|
||
|
|
# Map standard token types to short names, used in CSS class naming.
|
||
|
|
# If you add a new item, please be sure to run this file to perform
|
||
|
|
# a consistency check for duplicate values.
|
||
|
|
STANDARD_TYPES = {
|
||
|
|
Token: '',
|
||
|
|
|
||
|
|
Text: '',
|
||
|
|
Whitespace: 'w',
|
||
|
|
Escape: 'esc',
|
||
|
|
Error: 'err',
|
||
|
|
Other: 'x',
|
||
|
|
|
||
|
|
Keyword: 'k',
|
||
|
|
Keyword.Constant: 'kc',
|
||
|
|
Keyword.Declaration: 'kd',
|
||
|
|
Keyword.Namespace: 'kn',
|
||
|
|
Keyword.Pseudo: 'kp',
|
||
|
|
Keyword.Reserved: 'kr',
|
||
|
|
Keyword.Type: 'kt',
|
||
|
|
|
||
|
|
Name: 'n',
|
||
|
|
Name.Attribute: 'na',
|
||
|
|
Name.Builtin: 'nb',
|
||
|
|
Name.Builtin.Pseudo: 'bp',
|
||
|
|
Name.Class: 'nc',
|
||
|
|
Name.Constant: 'no',
|
||
|
|
Name.Decorator: 'nd',
|
||
|
|
Name.Entity: 'ni',
|
||
|
|
Name.Exception: 'ne',
|
||
|
|
Name.Function: 'nf',
|
||
|
|
Name.Function.Magic: 'fm',
|
||
|
|
Name.Property: 'py',
|
||
|
|
Name.Label: 'nl',
|
||
|
|
Name.Namespace: 'nn',
|
||
|
|
Name.Other: 'nx',
|
||
|
|
Name.Tag: 'nt',
|
||
|
|
Name.Variable: 'nv',
|
||
|
|
Name.Variable.Class: 'vc',
|
||
|
|
Name.Variable.Global: 'vg',
|
||
|
|
Name.Variable.Instance: 'vi',
|
||
|
|
Name.Variable.Magic: 'vm',
|
||
|
|
|
||
|
|
Literal: 'l',
|
||
|
|
Literal.Date: 'ld',
|
||
|
|
|
||
|
|
String: 's',
|
||
|
|
String.Affix: 'sa',
|
||
|
|
String.Backtick: 'sb',
|
||
|
|
String.Char: 'sc',
|
||
|
|
String.Delimiter: 'dl',
|
||
|
|
String.Doc: 'sd',
|
||
|
|
String.Double: 's2',
|
||
|
|
String.Escape: 'se',
|
||
|
|
String.Heredoc: 'sh',
|
||
|
|
String.Interpol: 'si',
|
||
|
|
String.Other: 'sx',
|
||
|
|
String.Regex: 'sr',
|
||
|
|
String.Single: 's1',
|
||
|
|
String.Symbol: 'ss',
|
||
|
|
|
||
|
|
Number: 'm',
|
||
|
|
Number.Bin: 'mb',
|
||
|
|
Number.Float: 'mf',
|
||
|
|
Number.Hex: 'mh',
|
||
|
|
Number.Integer: 'mi',
|
||
|
|
Number.Integer.Long: 'il',
|
||
|
|
Number.Oct: 'mo',
|
||
|
|
|
||
|
|
Operator: 'o',
|
||
|
|
Operator.Word: 'ow',
|
||
|
|
|
||
|
|
Punctuation: 'p',
|
||
|
|
Punctuation.Marker: 'pm',
|
||
|
|
|
||
|
|
Comment: 'c',
|
||
|
|
Comment.Hashbang: 'ch',
|
||
|
|
Comment.Multiline: 'cm',
|
||
|
|
Comment.Preproc: 'cp',
|
||
|
|
Comment.PreprocFile: 'cpf',
|
||
|
|
Comment.Single: 'c1',
|
||
|
|
Comment.Special: 'cs',
|
||
|
|
|
||
|
|
Generic: 'g',
|
||
|
|
Generic.Deleted: 'gd',
|
||
|
|
Generic.Emph: 'ge',
|
||
|
|
Generic.Error: 'gr',
|
||
|
|
Generic.Heading: 'gh',
|
||
|
|
Generic.Inserted: 'gi',
|
||
|
|
Generic.Output: 'go',
|
||
|
|
Generic.Prompt: 'gp',
|
||
|
|
Generic.Strong: 'gs',
|
||
|
|
Generic.Subheading: 'gu',
|
||
|
|
Generic.Traceback: 'gt',
|
||
|
|
}
|