tokens.py 1.7 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071
  1. #
  2. # Copyright (C) 2009-2020 the sqlparse authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of python-sqlparse and is released under
  6. # the BSD License: https://opensource.org/licenses/BSD-3-Clause
  7. #
  8. # The Token implementation is based on pygment's token system written
  9. # by Georg Brandl.
  10. # http://pygments.org/
  11. """Tokens"""
  12. class _TokenType(tuple):
  13. parent = None
  14. def __contains__(self, item):
  15. return item is not None and (self is item or item[:len(self)] == self)
  16. def __getattr__(self, name):
  17. # don't mess with dunder
  18. if name.startswith('__'):
  19. return super().__getattr__(self, name)
  20. new = _TokenType(self + (name,))
  21. setattr(self, name, new)
  22. new.parent = self
  23. return new
  24. def __repr__(self):
  25. # self can be False only if its the `root` i.e. Token itself
  26. return 'Token' + ('.' if self else '') + '.'.join(self)
  27. Token = _TokenType()
  28. # Special token types
  29. Text = Token.Text
  30. Whitespace = Text.Whitespace
  31. Newline = Whitespace.Newline
  32. Error = Token.Error
  33. # Text that doesn't belong to this lexer (e.g. HTML in PHP)
  34. Other = Token.Other
  35. # Common token types for source code
  36. Keyword = Token.Keyword
  37. Name = Token.Name
  38. Literal = Token.Literal
  39. String = Literal.String
  40. Number = Literal.Number
  41. Punctuation = Token.Punctuation
  42. Operator = Token.Operator
  43. Comparison = Operator.Comparison
  44. Wildcard = Token.Wildcard
  45. Comment = Token.Comment
  46. Assignment = Token.Assignment
  47. # Generic types for non-source code
  48. Generic = Token.Generic
  49. Command = Generic.Command
  50. # String and some others are not direct children of Token.
  51. # alias them:
  52. Token.Token = Token
  53. Token.String = String
  54. Token.Number = Number
  55. # SQL specific tokens
  56. DML = Keyword.DML
  57. DDL = Keyword.DDL
  58. CTE = Keyword.CTE