from __future__ import absolute_import from enum import Enum class TokenType: name: str contains_syntax: bool def __init__(self, name: str, contains_syntax: bool = False): self.name = name self.contains_syntax = contains_syntax def __repr__(self): return '%s(%s)' % (self.__class__.__name__, self.name) class PythonTokenTypes(Enum): STRING = TokenType('STRING') NUMBER = TokenType('NUMBER') NAME = TokenType('NAME', contains_syntax=True) ERRORTOKEN = TokenType('ERRORTOKEN') NEWLINE = TokenType('NEWLINE') INDENT = TokenType('INDENT') DEDENT = TokenType('DEDENT') ERROR_DEDENT = TokenType('ERROR_DEDENT') FSTRING_STRING = TokenType('FSTRING_STRING') FSTRING_START = TokenType('FSTRING_START') FSTRING_END = TokenType('FSTRING_END') OP = TokenType('OP', contains_syntax=True) ENDMARKER = TokenType('ENDMARKER')