tools/int_fiction.py

   1 # -*- coding: utf-8 -*-
   2 """
   3     pygments.lexers.int_fiction
   4     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
   5
   6     Lexers for interactive fiction languages.
   7
   8     :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
   9     :license: BSD, see LICENSE for details.
  10 """
  11
  12 import re
  13
  14 from pygments.lexer import RegexLexer, include, bygroups, using, \
  15     this, default, words
  16 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
  17     Number, Punctuation, Error, Generic
  18
  19 __all__ = ['Inform6Lexer', 'Inform6TemplateLexer', 'Inform7Lexer',
  20            'Tads3Lexer']
  21
  22
  23 class Inform6Lexer(RegexLexer):
  24     """
  25     For `Inform 6 <http://inform-fiction.org/>`_ source code.
  26
  27     .. versionadded:: 2.0
  28     """
  29
  30     name = 'Inform 6'
  31     aliases = ['inform6', 'i6']
  32     filenames = ['*.inf']
  33
  34     flags = re.MULTILINE | re.DOTALL | re.UNICODE
  35
  36     _name = r'[a-zA-Z_]\w*'
  37
  38     # Inform 7 maps these four character classes to their ASCII
  39     # equivalents. To support Inform 6 inclusions within Inform 7,
  40     # Inform6Lexer maps them too.
  41     _dash = u'\\-\u2010-\u2014'
  42     _dquote = u'"\u201c\u201d'
  43     _squote = u"'\u2018\u2019"
  44     _newline = u'\\n\u0085\u2028\u2029'
  45
  46     tokens = {
  47         'root': [
  48             (r'\A(!%%[^%s]*[%s])+' % (_newline, _newline), Comment.Preproc,
  49              'directive'),
  50             default('directive')
  51         ],
  52         '_whitespace': [
  53             (r'\s+', Text),
  54             (r'![^%s]*' % _newline, Comment.Single)
  55         ],
  56         'default': [
  57             include('_whitespace'),
  58             (r'\[', Punctuation, 'many-values'),  # Array initialization
  59             (r':|(?=;)', Punctuation, '#pop'),
  60             (r'<', Punctuation),  # Second angle bracket in an action statement
  61             default(('expression', '_expression'))
  62         ],
  63
  64         # Expressions
  65         '_expression': [
  66             include('_whitespace'),
  67             (r'(?=sp\b)', Text, '#pop'),
  68             (r'(?=[%s%s$0-9#a-zA-Z_])' % (_dquote, _squote), Text,
  69              ('#pop', 'value')),
  70             (r'\+\+|[%s]{1,2}(?!>)|~~?' % _dash, Operator),
  71             (r'(?=[()\[%s,?@{:;])' % _dash, Text, '#pop')
  72         ],
  73         'expression': [
  74             include('_whitespace'),
  75             (r'\(', Punctuation, ('expression', '_expression')),
  76             (r'\)', Punctuation, '#pop'),
  77             (r'\[', Punctuation, ('#pop', 'statements', 'locals')),
  78             (r'>(?=(\s+|(![^%s]*))*[>;])' % _newline, Punctuation),
  79             (r'\+\+|[%s]{2}(?!>)' % _dash, Operator),
  80             (r',', Punctuation, '_expression'),
  81             (r'&&?|\|\|?|[=~><]?=|[%s]{1,2}>?|\.\.?[&#]?|::|[<>+*/%%]' % _dash,
  82              Operator, '_expression'),
  83             (r'(has|hasnt|in|notin|ofclass|or|provides)\b', Operator.Word,
  84              '_expression'),
  85             (r'sp\b', Name),
  86             (r'\?~?', Name.Label, 'label?'),
  87             (r'[@{]', Error),
  88             default('#pop')
  89         ],
  90         '_assembly-expression': [
  91             (r'\(', Punctuation, ('#push', '_expression')),
  92             (r'[\[\]]', Punctuation),
  93             (r'[%s]>' % _dash, Punctuation, '_expression'),
  94             (r'sp\b', Keyword.Pseudo),
  95             (r';', Punctuation, '#pop:3'),
  96             include('expression')
  97         ],
  98         '_for-expression': [
  99             (r'\)', Punctuation, '#pop:2'),
 100             (r':', Punctuation, '#pop'),
 101             include('expression')
 102         ],
 103         '_keyword-expression': [
 104             (r'(from|near|to)\b', Keyword, '_expression'),
 105             include('expression')
 106         ],
 107         '_list-expression': [
 108             (r',', Punctuation, '#pop'),
 109             include('expression')
 110         ],
 111         '_object-expression': [
 112             (r'has\b', Keyword.Declaration, '#pop'),
 113             include('_list-expression')
 114         ],
 115
 116         # Values
 117         'value': [
 118             include('_whitespace'),
 119             # Strings
 120             (r'[%s][^@][%s]' % (_squote, _squote), String.Char, '#pop'),
 121             (r'([%s])(@\{[0-9a-fA-F]{1,4}\})([%s])' % (_squote, _squote),
 122              bygroups(String.Char, String.Escape, String.Char), '#pop'),
 123             (r'([%s])(@.{2})([%s])' % (_squote, _squote),
 124              bygroups(String.Char, String.Escape, String.Char), '#pop'),
 125             (r'[%s]' % _squote, String.Single, ('#pop', 'dictionary-word')),
 126             (r'[%s]' % _dquote, String.Double, ('#pop', 'string')),
 127             # Numbers
 128             (r'\$[+%s][0-9]*\.?[0-9]*([eE][+%s]?[0-9]+)?' % (_dash, _dash),
 129              Number.Float, '#pop'),
 130             (r'\$[0-9a-fA-F]+', Number.Hex, '#pop'),
 131             (r'\$\$[01]+', Number.Bin, '#pop'),
 132             (r'[0-9]+', Number.Integer, '#pop'),
 133             # Values prefixed by hashes
 134             (r'(##|#a\$)(%s)' % _name, bygroups(Operator, Name), '#pop'),
 135             (r'(#g\$)(%s)' % _name,
 136              bygroups(Operator, Name.Variable.Global), '#pop'),
 137             (r'#[nw]\$', Operator, ('#pop', 'obsolete-dictionary-word')),
 138             (r'(#r\$)(%s)' % _name, bygroups(Operator, Name.Function), '#pop'),
 139             (r'#', Name.Builtin, ('#pop', 'system-constant')),
 140             # System functions
 141             (words((
 142                 'child', 'children', 'elder', 'eldest', 'glk', 'indirect', 'metaclass',
 143                 'parent', 'random', 'sibling', 'younger', 'youngest'), suffix=r'\b'),
 144              Name.Builtin, '#pop'),
 145             # Metaclasses
 146             (r'(?i)(Class|Object|Routine|String)\b', Name.Builtin, '#pop'),
 147             # Veneer routines
 148             (words((
 149                 'Box__Routine', 'CA__Pr', 'CDefArt', 'CInDefArt', 'Cl__Ms',
 150                 'Copy__Primitive', 'CP__Tab', 'DA__Pr', 'DB__Pr', 'DefArt', 'Dynam__String',
 151                 'EnglishNumber', 'Glk__Wrap', 'IA__Pr', 'IB__Pr', 'InDefArt', 'Main__',
 152                 'Meta__class', 'OB__Move', 'OB__Remove', 'OC__Cl', 'OP__Pr', 'Print__Addr',
 153                 'Print__PName', 'PrintShortName', 'RA__Pr', 'RA__Sc', 'RL__Pr', 'R_Process',
 154                 'RT__ChG', 'RT__ChGt', 'RT__ChLDB', 'RT__ChLDW', 'RT__ChPR', 'RT__ChPrintA',
 155                 'RT__ChPrintC', 'RT__ChPrintO', 'RT__ChPrintS', 'RT__ChPS', 'RT__ChR',
 156                 'RT__ChSTB', 'RT__ChSTW', 'RT__ChT', 'RT__Err', 'RT__TrPS', 'RV__Pr',
 157                 'Symb__Tab', 'Unsigned__Compare', 'WV__Pr', 'Z__Region'),
 158                 prefix='(?i)', suffix=r'\b'),
 159              Name.Builtin, '#pop'),
 160             # Other built-in symbols
 161             (words((
 162                 'call', 'copy', 'create', 'DEBUG', 'destroy', 'DICT_CHAR_SIZE',
 163                 'DICT_ENTRY_BYTES', 'DICT_IS_UNICODE', 'DICT_WORD_SIZE', 'false',
 164                 'FLOAT_INFINITY', 'FLOAT_NAN', 'FLOAT_NINFINITY', 'GOBJFIELD_CHAIN',
 165                 'GOBJFIELD_CHILD', 'GOBJFIELD_NAME', 'GOBJFIELD_PARENT',
 166                 'GOBJFIELD_PROPTAB', 'GOBJFIELD_SIBLING', 'GOBJ_EXT_START',
 167                 'GOBJ_TOTAL_LENGTH', 'Grammar__Version', 'INDIV_PROP_START', 'INFIX',
 168                 'infix__watching', 'MODULE_MODE', 'name', 'nothing', 'NUM_ATTR_BYTES', 'print',
 169                 'print_to_array', 'recreate', 'remaining', 'self', 'sender', 'STRICT_MODE',
 170                 'sw__var', 'sys__glob0', 'sys__glob1', 'sys__glob2', 'sys_statusline_flag',
 171                 'TARGET_GLULX', 'TARGET_ZCODE', 'temp__global2', 'temp__global3',
 172                 'temp__global4', 'temp_global', 'true', 'USE_MODULES', 'WORDSIZE'),
 173                 prefix='(?i)', suffix=r'\b'),
 174              Name.Builtin, '#pop'),
 175             # Other values
 176             (_name, Name, '#pop')
 177         ],
 178         # Strings
 179         'dictionary-word': [
 180             (r'[~^]+', String.Escape),
 181             (r'[^~^\\@({%s]+' % _squote, String.Single),
 182             (r'[({]', String.Single),
 183             (r'@\{[0-9a-fA-F]{,4}\}', String.Escape),
 184             (r'@.{2}', String.Escape),
 185             (r'[%s]' % _squote, String.Single, '#pop')
 186         ],
 187         'string': [
 188             (r'[~^]+', String.Escape),
 189             (r'[^~^\\@({%s]+' % _dquote, String.Double),
 190             (r'[({]', String.Double),
 191             (r'\\', String.Escape),
 192             (r'@(\\\s*[%s]\s*)*@((\\\s*[%s]\s*)*[0-9])*' %
 193              (_newline, _newline), String.Escape),
 194             (r'@(\\\s*[%s]\s*)*\{((\\\s*[%s]\s*)*[0-9a-fA-F]){,4}'
 195              r'(\\\s*[%s]\s*)*\}' % (_newline, _newline, _newline),
 196              String.Escape),
 197             (r'@(\\\s*[%s]\s*)*.(\\\s*[%s]\s*)*.' % (_newline, _newline),
 198              String.Escape),
 199             (r'[%s]' % _dquote, String.Double, '#pop')
 200         ],
 201         'plain-string': [
 202             (r'[^~^\\({\[\]%s]+' % _dquote, String.Double),
 203             (r'[~^({\[\]]', String.Double),
 204             (r'\\', String.Escape),
 205             (r'[%s]' % _dquote, String.Double, '#pop')
 206         ],
 207         # Names
 208         '_constant': [
 209             include('_whitespace'),
 210             (_name, Name.Constant, '#pop'),
 211             include('value')
 212         ],
 213         '_global': [
 214             include('_whitespace'),
 215             (_name, Name.Variable.Global, '#pop'),
 216             include('value')
 217         ],
 218         'label?': [
 219             include('_whitespace'),
 220             (_name, Name.Label, '#pop'),
 221             default('#pop')
 222         ],
 223         'variable?': [
 224             include('_whitespace'),
 225             (_name, Name.Variable, '#pop'),
 226             default('#pop')
 227         ],
 228         # Values after hashes
 229         'obsolete-dictionary-word': [
 230             (r'\S\w*', String.Other, '#pop')
 231         ],
 232         'system-constant': [
 233             include('_whitespace'),
 234             (_name, Name.Builtin, '#pop')
 235         ],
 236
 237         # Directives
 238         'directive': [
 239             include('_whitespace'),
 240             (r'#', Punctuation),
 241             (r';', Punctuation, '#pop'),
 242             (r'\[', Punctuation,
 243              ('default', 'statements', 'locals', 'routine-name?')),
 244             (words((
 245                 'abbreviate', 'endif', 'dictionary', 'ifdef', 'iffalse', 'ifndef', 'ifnot',
 246                 'iftrue', 'ifv3', 'ifv5', 'release', 'serial', 'switches', 'system_file',
 247                 'version'), prefix='(?i)', suffix=r'\b'),
 248              Keyword, 'default'),
 249             (r'(?i)(array|global)\b', Keyword,
 250              ('default', 'directive-keyword?', '_global')),
 251             (r'(?i)attribute\b', Keyword, ('default', 'alias?', '_constant')),
 252             (r'(?i)class\b', Keyword,
 253              ('object-body', 'duplicates', 'class-name')),
 254             (r'(?i)(constant|default)\b', Keyword,
 255              ('default', 'expression', '_constant')),
 256             (r'(?i)(end\b)(.*)', bygroups(Keyword, Text)),
 257             (r'(?i)(extend|verb)\b', Keyword, 'grammar'),
 258             (r'(?i)fake_action\b', Keyword, ('default', '_constant')),
 259             (r'(?i)import\b', Keyword, 'manifest'),
 260             (r'(?i)(include|link)\b', Keyword,
 261              ('default', 'before-plain-string')),
 262             (r'(?i)(lowstring|undef)\b', Keyword, ('default', '_constant')),
 263             (r'(?i)message\b', Keyword, ('default', 'diagnostic')),
 264             (r'(?i)(nearby|object)\b', Keyword,
 265              ('object-body', '_object-head')),
 266             (r'(?i)property\b', Keyword,
 267              ('default', 'alias?', '_constant', 'property-keyword*')),
 268             (r'(?i)replace\b', Keyword,
 269              ('default', 'routine-name?', 'routine-name?')),
 270             (r'(?i)statusline\b', Keyword, ('default', 'directive-keyword?')),
 271             (r'(?i)stub\b', Keyword, ('default', 'routine-name?')),
 272             (r'(?i)trace\b', Keyword,
 273              ('default', 'trace-keyword?', 'trace-keyword?')),
 274             (r'(?i)zcharacter\b', Keyword,
 275              ('default', 'directive-keyword?', 'directive-keyword?')),
 276             (_name, Name.Class, ('object-body', '_object-head'))
 277         ],
 278         # [, Replace, Stub
 279         'routine-name?': [
 280             include('_whitespace'),
 281             (_name, Name.Function, '#pop'),
 282             default('#pop')
 283         ],
 284         'locals': [
 285             include('_whitespace'),
 286             (r';', Punctuation, '#pop'),
 287             (r'\*', Punctuation),
 288             (_name, Name.Variable)
 289         ],
 290         # Array
 291         'many-values': [
 292             include('_whitespace'),
 293             (r';', Punctuation),
 294             (r'\]', Punctuation, '#pop'),
 295             (r':', Error),
 296             default(('expression', '_expression'))
 297         ],
 298         # Attribute, Property
 299         'alias?': [
 300             include('_whitespace'),
 301             (r'alias\b', Keyword, ('#pop', '_constant')),
 302             default('#pop')
 303         ],
 304         # Class, Object, Nearby
 305         'class-name': [
 306             include('_whitespace'),
 307             (r'(?=[,;]|(class|has|private|with)\b)', Text, '#pop'),
 308             (_name, Name.Class, '#pop')
 309         ],
 310         'duplicates': [
 311             include('_whitespace'),
 312             (r'\(', Punctuation, ('#pop', 'expression', '_expression')),
 313             default('#pop')
 314         ],
 315         '_object-head': [
 316             (r'[%s]>' % _dash, Punctuation),
 317             (r'(class|has|private|with)\b', Keyword.Declaration, '#pop'),
 318             include('_global')
 319         ],
 320         'object-body': [
 321             include('_whitespace'),
 322             (r';', Punctuation, '#pop:2'),
 323             (r',', Punctuation),
 324             (r'class\b', Keyword.Declaration, 'class-segment'),
 325             (r'(has|private|with)\b', Keyword.Declaration),
 326             (r':', Error),
 327             default(('_object-expression', '_expression'))
 328         ],
 329         'class-segment': [
 330             include('_whitespace'),
 331             (r'(?=[,;]|(class|has|private|with)\b)', Text, '#pop'),
 332             (_name, Name.Class),
 333             default('value')
 334         ],
 335         # Extend, Verb
 336         'grammar': [
 337             include('_whitespace'),
 338             (r'=', Punctuation, ('#pop', 'default')),
 339             (r'\*', Punctuation, ('#pop', 'grammar-line')),
 340             default('_directive-keyword')
 341         ],
 342         'grammar-line': [
 343             include('_whitespace'),
 344             (r';', Punctuation, '#pop'),
 345             (r'[/*]', Punctuation),
 346             (r'[%s]>' % _dash, Punctuation, 'value'),
 347             (r'(noun|scope)\b', Keyword, '=routine'),
 348             default('_directive-keyword')
 349         ],
 350         '=routine': [
 351             include('_whitespace'),
 352             (r'=', Punctuation, 'routine-name?'),
 353             default('#pop')
 354         ],
 355         # Import
 356         'manifest': [
 357             include('_whitespace'),
 358             (r';', Punctuation, '#pop'),
 359             (r',', Punctuation),
 360             (r'(?i)global\b', Keyword, '_global'),
 361             default('_global')
 362         ],
 363         # Include, Link, Message
 364         'diagnostic': [
 365             include('_whitespace'),
 366             (r'[%s]' % _dquote, String.Double, ('#pop', 'message-string')),
 367             default(('#pop', 'before-plain-string', 'directive-keyword?'))
 368         ],
 369         'before-plain-string': [
 370             include('_whitespace'),
 371             (r'[%s]' % _dquote, String.Double, ('#pop', 'plain-string'))
 372         ],
 373         'message-string': [
 374             (r'[~^]+', String.Escape),
 375             include('plain-string')
 376         ],
 377
 378         # Keywords used in directives
 379         '_directive-keyword!': [
 380             include('_whitespace'),
 381             (words((
 382                 'additive', 'alias', 'buffer', 'class', 'creature', 'data', 'error', 'fatalerror',
 383                 'first', 'has', 'held', 'initial', 'initstr', 'last', 'long', 'meta', 'multi',
 384                 'multiexcept', 'multiheld', 'multiinside', 'noun', 'number', 'only', 'private',
 385                 'replace', 'reverse', 'scope', 'score', 'special', 'string', 'table', 'terminating',
 386                 'time', 'topic', 'warning', 'with'), suffix=r'\b'),
 387              Keyword, '#pop'),
 388             (r'[%s]{1,2}>|[+=]' % _dash, Punctuation, '#pop')
 389         ],
 390         '_directive-keyword': [
 391             include('_directive-keyword!'),
 392             include('value')
 393         ],
 394         'directive-keyword?': [
 395             include('_directive-keyword!'),
 396             default('#pop')
 397         ],
 398         'property-keyword*': [
 399             include('_whitespace'),
 400             (r'(additive|long)\b', Keyword),
 401             default('#pop')
 402         ],
 403         'trace-keyword?': [
 404             include('_whitespace'),
 405             (words((
 406                 'assembly', 'dictionary', 'expressions', 'lines', 'linker',
 407                 'objects', 'off', 'on', 'symbols', 'tokens', 'verbs'), suffix=r'\b'),
 408              Keyword, '#pop'),
 409             default('#pop')
 410         ],
 411
 412         # Statements
 413         'statements': [
 414             include('_whitespace'),
 415             (r'\]', Punctuation, '#pop'),
 416             (r'[;{}]', Punctuation),
 417             (words((
 418                 'box', 'break', 'continue', 'default', 'give', 'inversion',
 419                 'new_line', 'quit', 'read', 'remove', 'return', 'rfalse', 'rtrue',
 420                 'spaces', 'string', 'until'), suffix=r'\b'),
 421              Keyword, 'default'),
 422             (r'(do|else)\b', Keyword),
 423             (r'(font|style)\b', Keyword,
 424              ('default', 'miscellaneous-keyword?')),
 425             (r'for\b', Keyword, ('for', '(?')),
 426             (r'(if|switch|while)', Keyword,
 427              ('expression', '_expression', '(?')),
 428             (r'(jump|save|restore)\b', Keyword, ('default', 'label?')),
 429             (r'objectloop\b', Keyword,
 430              ('_keyword-expression', 'variable?', '(?')),
 431             (r'print(_ret)?\b|(?=[%s])' % _dquote, Keyword, 'print-list'),
 432             (r'\.', Name.Label, 'label?'),
 433             (r'@', Keyword, 'opcode'),
 434             (r'#(?![agrnw]\$|#)', Punctuation, 'directive'),
 435             (r'<', Punctuation, 'default'),
 436             (r'move\b', Keyword,
 437              ('default', '_keyword-expression', '_expression')),
 438             default(('default', '_keyword-expression', '_expression'))
 439         ],
 440         'miscellaneous-keyword?': [
 441             include('_whitespace'),
 442             (r'(bold|fixed|from|near|off|on|reverse|roman|to|underline)\b',
 443              Keyword, '#pop'),
 444             (r'(a|A|an|address|char|name|number|object|property|string|the|'
 445              r'The)\b(?=(\s+|(![^%s]*))*\))' % _newline, Keyword.Pseudo,
 446              '#pop'),
 447             (r'%s(?=(\s+|(![^%s]*))*\))' % (_name, _newline), Name.Function,
 448              '#pop'),
 449             default('#pop')
 450         ],
 451         '(?': [
 452             include('_whitespace'),
 453             (r'\(', Punctuation, '#pop'),
 454             default('#pop')
 455         ],
 456         'for': [
 457             include('_whitespace'),
 458             (r';', Punctuation, ('_for-expression', '_expression')),
 459             default(('_for-expression', '_expression'))
 460         ],
 461         'print-list': [
 462             include('_whitespace'),
 463             (r';', Punctuation, '#pop'),
 464             (r':', Error),
 465             default(('_list-expression', '_expression', '_list-expression', 'form'))
 466         ],
 467         'form': [
 468             include('_whitespace'),
 469             (r'\(', Punctuation, ('#pop', 'miscellaneous-keyword?')),
 470             default('#pop')
 471         ],
 472
 473         # Assembly
 474         'opcode': [
 475             include('_whitespace'),
 476             (r'[%s]' % _dquote, String.Double, ('operands', 'plain-string')),
 477             (_name, Keyword, 'operands')
 478         ],
 479         'operands': [
 480             (r':', Error),
 481             default(('_assembly-expression', '_expression'))
 482         ]
 483     }
 484
 485     def get_tokens_unprocessed(self, text):
 486         # 'in' is either a keyword or an operator.
 487         # If the token two tokens after 'in' is ')', 'in' is a keyword:
 488         #   objectloop(a in b)
 489         # Otherwise, it is an operator:
 490         #   objectloop(a in b && true)
 491         objectloop_queue = []
 492         objectloop_token_count = -1
 493         previous_token = None
 494         for index, token, value in RegexLexer.get_tokens_unprocessed(self,
 495                                                                      text):
 496             if previous_token is Name.Variable and value == 'in':
 497                 objectloop_queue = [[index, token, value]]
 498                 objectloop_token_count = 2
 499             elif objectloop_token_count > 0:
 500                 if token not in Comment and token not in Text:
 501                     objectloop_token_count -= 1
 502                 objectloop_queue.append((index, token, value))
 503             else:
 504                 if objectloop_token_count == 0:
 505                     if objectloop_queue[-1][2] == ')':
 506                         objectloop_queue[0][1] = Keyword
 507                     while objectloop_queue:
 508                         yield objectloop_queue.pop(0)
 509                     objectloop_token_count = -1
 510                 yield index, token, value
 511             if token not in Comment and token not in Text:
 512                 previous_token = token
 513         while objectloop_queue:
 514             yield objectloop_queue.pop(0)
 515
 516
 517 class Inform7Lexer(RegexLexer):
 518     """
 519     For `Inform 7 <http://inform7.com/>`_ source code.
 520
 521     .. versionadded:: 2.0
 522     """
 523
 524     name = 'Inform 7'
 525     aliases = ['inform7', 'i7']
 526     filenames = ['*.ni', '*.i7x']
 527
 528     flags = re.MULTILINE | re.DOTALL | re.UNICODE
 529
 530     _dash = Inform6Lexer._dash
 531     _dquote = Inform6Lexer._dquote
 532     _newline = Inform6Lexer._newline
 533     _start = r'\A|(?<=[%s])' % _newline
 534
 535     # There are three variants of Inform 7, differing in how to
 536     # interpret at signs and braces in I6T. In top-level inclusions, at
 537     # signs in the first column are inweb syntax. In phrase definitions
 538     # and use options, tokens in braces are treated as I7. Use options
 539     # also interpret "{N}".
 540     tokens = {}
 541     token_variants = ['+i6t-not-inline', '+i6t-inline', '+i6t-use-option']
 542
 543     for level in token_variants:
 544         tokens[level] = {
 545             '+i6-root': list(Inform6Lexer.tokens['root']),
 546             '+i6t-root': [  # For Inform6TemplateLexer
 547                 (r'[^%s]*' % Inform6Lexer._newline, Comment.Preproc,
 548                  ('directive', '+p'))
 549             ],
 550             'root': [
 551                 (r'(\|?\s)+', Text),
 552                 (r'\[', Comment.Multiline, '+comment'),
 553                 (r'[%s]' % _dquote, Generic.Heading,
 554                  ('+main', '+titling', '+titling-string')),
 555                 default(('+main', '+heading?'))
 556             ],
 557             '+titling-string': [
 558                 (r'[^%s]+' % _dquote, Generic.Heading),
 559                 (r'[%s]' % _dquote, Generic.Heading, '#pop')
 560             ],
 561             '+titling': [
 562                 (r'\[', Comment.Multiline, '+comment'),
 563                 (r'[^%s.;:|%s]+' % (_dquote, _newline), Generic.Heading),
 564                 (r'[%s]' % _dquote, Generic.Heading, '+titling-string'),
 565                 (r'[%s]{2}|(?<=[\s%s])\|[\s%s]' % (_newline, _dquote, _dquote),
 566                  Text, ('#pop', '+heading?')),
 567                 (r'[.;:]|(?<=[\s%s])\|' % _dquote, Text, '#pop'),
 568                 (r'[|%s]' % _newline, Generic.Heading)
 569             ],
 570             '+main': [
 571                 (r'(?i)[^%s:a\[(|%s]+' % (_dquote, _newline), Text),
 572                 (r'[%s]' % _dquote, String.Double, '+text'),
 573                 (r':', Text, '+phrase-definition'),
 574                 (r'(?i)\bas\b', Text, '+use-option'),
 575                 (r'\[', Comment.Multiline, '+comment'),
 576                 (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash),
 577                  bygroups(Punctuation,
 578                           using(this, state=('+i6-root', 'directive'),
 579                                 i6t='+i6t-not-inline'), Punctuation)),
 580                 (r'(%s|(?<=[\s;:.%s]))\|\s|[%s]{2,}' %
 581                  (_start, _dquote, _newline), Text, '+heading?'),
 582                 (r'(?i)[a(|%s]' % _newline, Text)
 583             ],
 584             '+phrase-definition': [
 585                 (r'\s+', Text),
 586                 (r'\[', Comment.Multiline, '+comment'),
 587                 (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash),
 588                  bygroups(Punctuation,
 589                           using(this, state=('+i6-root', 'directive',
 590                                              'default', 'statements'),
 591                                 i6t='+i6t-inline'), Punctuation), '#pop'),
 592                 default('#pop')
 593             ],
 594             '+use-option': [
 595                 (r'\s+', Text),
 596                 (r'\[', Comment.Multiline, '+comment'),
 597                 (r'(\([%s])(.*?)([%s]\))' % (_dash, _dash),
 598                  bygroups(Punctuation,
 599                           using(this, state=('+i6-root', 'directive'),
 600                                 i6t='+i6t-use-option'), Punctuation), '#pop'),
 601                 default('#pop')
 602             ],
 603             '+comment': [
 604                 (r'[^\[\]]+', Comment.Multiline),
 605                 (r'\[', Comment.Multiline, '#push'),
 606                 (r'\]', Comment.Multiline, '#pop')
 607             ],
 608             '+text': [
 609                 (r'[^\[%s]+' % _dquote, String.Double),
 610                 (r'\[.*?\]', String.Interpol),
 611                 (r'[%s]' % _dquote, String.Double, '#pop')
 612             ],
 613             '+heading?': [
 614                 (r'(\|?\s)+', Text),
 615                 (r'\[', Comment.Multiline, '+comment'),
 616                 (r'[%s]{4}\s+' % _dash, Text, '+documentation-heading'),
 617                 (r'[%s]{1,3}' % _dash, Text),
 618                 (r'(?i)(volume|book|part|chapter|section)\b[^%s]*' % _newline,
 619                  Generic.Heading, '#pop'),
 620                 default('#pop')
 621             ],
 622             '+documentation-heading': [
 623                 (r'\s+', Text),
 624                 (r'\[', Comment.Multiline, '+comment'),
 625                 (r'(?i)documentation\s+', Text, '+documentation-heading2'),
 626                 default('#pop')
 627             ],
 628             '+documentation-heading2': [
 629                 (r'\s+', Text),
 630                 (r'\[', Comment.Multiline, '+comment'),
 631                 (r'[%s]{4}\s' % _dash, Text, '+documentation'),
 632                 default('#pop:2')
 633             ],
 634             '+documentation': [
 635                 (r'(?i)(%s)\s*(chapter|example)\s*:[^%s]*' %
 636                  (_start, _newline), Generic.Heading),
 637                 (r'(?i)(%s)\s*section\s*:[^%s]*' % (_start, _newline),
 638                  Generic.Subheading),
 639                 (r'((%s)\t.*?[%s])+' % (_start, _newline),
 640                  using(this, state='+main')),
 641                 (r'[^%s\[]+|[%s\[]' % (_newline, _newline), Text),
 642                 (r'\[', Comment.Multiline, '+comment'),
 643             ],
 644             '+i6t-not-inline': [
 645                 (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline),
 646                  Comment.Preproc),
 647                 (r'(%s)@([%s]+|Purpose:)[^%s]*' % (_start, _dash, _newline),
 648                  Comment.Preproc),
 649                 (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline),
 650                  Generic.Heading, '+p')
 651             ],
 652             '+i6t-use-option': [
 653                 include('+i6t-not-inline'),
 654                 (r'(\{)(N)(\})', bygroups(Punctuation, Text, Punctuation))
 655             ],
 656             '+i6t-inline': [
 657                 (r'(\{)(\S[^}]*)?(\})',
 658                  bygroups(Punctuation, using(this, state='+main'),
 659                           Punctuation))
 660             ],
 661             '+i6t': [
 662                 (r'(\{[%s])(![^}]*)(\}?)' % _dash,
 663                  bygroups(Punctuation, Comment.Single, Punctuation)),
 664                 (r'(\{[%s])(lines)(:)([^}]*)(\}?)' % _dash,
 665                  bygroups(Punctuation, Keyword, Punctuation, Text,
 666                           Punctuation), '+lines'),
 667                 (r'(\{[%s])([^:}]*)(:?)([^}]*)(\}?)' % _dash,
 668                  bygroups(Punctuation, Keyword, Punctuation, Text,
 669                           Punctuation)),
 670                 (r'(\(\+)(.*?)(\+\)|\Z)',
 671                  bygroups(Punctuation, using(this, state='+main'),
 672                           Punctuation))
 673             ],
 674             '+p': [
 675                 (r'[^@]+', Comment.Preproc),
 676                 (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline),
 677                  Comment.Preproc, '#pop'),
 678                 (r'(%s)@([%s]|Purpose:)' % (_start, _dash), Comment.Preproc),
 679                 (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline),
 680                  Generic.Heading),
 681                 (r'@', Comment.Preproc)
 682             ],
 683             '+lines': [
 684                 (r'(%s)@c( .*?)?([%s]|\Z)' % (_start, _newline),
 685                  Comment.Preproc),
 686                 (r'(%s)@([%s]|Purpose:)[^%s]*' % (_start, _dash, _newline),
 687                  Comment.Preproc),
 688                 (r'(%s)@p( .*?)?([%s]|\Z)' % (_start, _newline),
 689                  Generic.Heading, '+p'),
 690                 (r'(%s)@\w*[ %s]' % (_start, _newline), Keyword),
 691                 (r'![^%s]*' % _newline, Comment.Single),
 692                 (r'(\{)([%s]endlines)(\})' % _dash,
 693                  bygroups(Punctuation, Keyword, Punctuation), '#pop'),
 694                 (r'[^@!{]+?([%s]|\Z)|.' % _newline, Text)
 695             ]
 696         }
 697         # Inform 7 can include snippets of Inform 6 template language,
 698         # so all of Inform6Lexer's states are copied here, with
 699         # modifications to account for template syntax. Inform7Lexer's
 700         # own states begin with '+' to avoid name conflicts. Some of
 701         # Inform6Lexer's states begin with '_': these are not modified.
 702         # They deal with template syntax either by including modified
 703         # states, or by matching r'' then pushing to modified states.
 704         for token in Inform6Lexer.tokens:
 705             if token == 'root':
 706                 continue
 707             tokens[level][token] = list(Inform6Lexer.tokens[token])
 708             if not token.startswith('_'):
 709                 tokens[level][token][:0] = [include('+i6t'), include(level)]
 710
 711     def __init__(self, **options):
 712         level = options.get('i6t', '+i6t-not-inline')
 713         if level not in self._all_tokens:
 714             self._tokens = self.__class__.process_tokendef(level)
 715         else:
 716             self._tokens = self._all_tokens[level]
 717         RegexLexer.__init__(self, **options)
 718
 719
 720 class Inform6TemplateLexer(Inform7Lexer):
 721     """
 722     For `Inform 6 template
 723     <http://inform7.com/sources/src/i6template/Woven/index.html>`_ code.
 724
 725     .. versionadded:: 2.0
 726     """
 727
 728     name = 'Inform 6 template'
 729     aliases = ['i6t']
 730     filenames = ['*.i6t']
 731
 732     def get_tokens_unprocessed(self, text, stack=('+i6t-root',)):
 733         return Inform7Lexer.get_tokens_unprocessed(self, text, stack)
 734
 735
 736 class Tads3Lexer(RegexLexer):
 737     """
 738     For `TADS 3 <http://www.tads.org/>`_ source code.
 739     """
 740
 741     name = 'TADS 3'
 742     aliases = ['tads3']
 743     filenames = ['*.t']
 744
 745     flags = re.DOTALL | re.MULTILINE
 746
 747     _comment_single = r'(?://(?:[^\\\n]|\\+[\w\W])*$)'
 748     _comment_multiline = r'(?:/\*(?:[^*]|\*(?!/))*\*/)'
 749     _escape = (r'(?:\\(?:[\n\\<>"\'^v bnrt]|u[\da-fA-F]{,4}|x[\da-fA-F]{,2}|'
 750                r'[0-3]?[0-7]{1,2}))')
 751     _name = r'(?:[_a-zA-Z]\w*)'
 752     _no_quote = r'(?=\s|\\?>)'
 753     _operator = (r'(?:&&|\|\||\+\+|--|\?\?|::|[.,@\[\]~]|'
 754                  r'(?:[=+\-*/%!&|^]|<<?|>>?>?)=?)')
 755     _ws = r'(?:\\|\s|%s|%s)' % (_comment_single, _comment_multiline)
 756     _ws_pp = r'(?:\\\n|[^\S\n]|%s|%s)' % (_comment_single, _comment_multiline)
 757
 758     def _make_string_state(triple, double, verbatim=None, _escape=_escape):
 759         if verbatim:
 760             verbatim = ''.join(['(?:%s|%s)' % (re.escape(c.lower()),
 761                                                re.escape(c.upper()))
 762                                 for c in verbatim])
 763         char = r'"' if double else r"'"
 764         token = String.Double if double else String.Single
 765         escaped_quotes = r'+|%s(?!%s{2})' % (char, char) if triple else r''
 766         prefix = '%s%s' % ('t' if triple else '', 'd' if double else 's')
 767         tag_state_name = '%sqt' % prefix
 768         state = []
 769         if triple:
 770             state += [
 771                 (r'%s{3,}' % char, token, '#pop'),
 772                 (r'\\%s+' % char, String.Escape),
 773                 (char, token)
 774             ]
 775         else:
 776             state.append((char, token, '#pop'))
 777         state += [
 778             include('s/verbatim'),
 779             (r'[^\\<&{}%s]+' % char, token)
 780         ]
 781         if verbatim:
 782             # This regex can't use `(?i)` because escape sequences are
 783             # case-sensitive. `<\XMP>` works; `<\xmp>` doesn't.
 784             state.append((r'\\?<(/|\\\\|(?!%s)\\)%s(?=[\s=>])' %
 785                           (_escape, verbatim),
 786                           Name.Tag, ('#pop', '%sqs' % prefix, tag_state_name)))
 787         else:
 788             state += [
 789                 (r'\\?<!([^><\\%s]|<(?!<)|\\%s%s|%s|\\.)*>?' %
 790                  (char, char, escaped_quotes, _escape), Comment.Multiline),
 791                 (r'(?i)\\?<listing(?=[\s=>]|\\>)', Name.Tag,
 792                  ('#pop', '%sqs/listing' % prefix, tag_state_name)),
 793                 (r'(?i)\\?<xmp(?=[\s=>]|\\>)', Name.Tag,
 794                  ('#pop', '%sqs/xmp' % prefix, tag_state_name)),
 795                 (r'\\?<([^\s=><\\%s]|<(?!<)|\\%s%s|%s|\\.)*' %
 796                  (char, char, escaped_quotes, _escape), Name.Tag,
 797                  tag_state_name),
 798                 include('s/entity')
 799             ]
 800         state += [
 801             include('s/escape'),
 802             (r'\{([^}<\\%s]|<(?!<)|\\%s%s|%s|\\.)*\}' %
 803              (char, char, escaped_quotes, _escape), String.Interpol),
 804             (r'[\\&{}<]', token)
 805         ]
 806         return state
 807
 808     def _make_tag_state(triple, double, _escape=_escape):
 809         char = r'"' if double else r"'"
 810         quantifier = r'{3,}' if triple else r''
 811         state_name = '%s%sqt' % ('t' if triple else '', 'd' if double else 's')
 812         token = String.Double if double else String.Single
 813         escaped_quotes = r'+|%s(?!%s{2})' % (char, char) if triple else r''
 814         return [
 815             (r'%s%s' % (char, quantifier), token, '#pop:2'),
 816             (r'(\s|\\\n)+', Text),
 817             (r'(=)(\\?")', bygroups(Punctuation, String.Double),
 818              'dqs/%s' % state_name),
 819             (r"(=)(\\?')", bygroups(Punctuation, String.Single),
 820              'sqs/%s' % state_name),
 821             (r'=', Punctuation, 'uqs/%s' % state_name),
 822             (r'\\?>', Name.Tag, '#pop'),
 823             (r'\{([^}<\\%s]|<(?!<)|\\%s%s|%s|\\.)*\}' %
 824              (char, char, escaped_quotes, _escape), String.Interpol),
 825             (r'([^\s=><\\%s]|<(?!<)|\\%s%s|%s|\\.)+' %
 826              (char, char, escaped_quotes, _escape), Name.Attribute),
 827             include('s/escape'),
 828             include('s/verbatim'),
 829             include('s/entity'),
 830             (r'[\\{}&]', Name.Attribute)
 831         ]
 832
 833     def _make_attribute_value_state(terminator, host_triple, host_double,
 834                                     _escape=_escape):
 835         token = (String.Double if terminator == r'"' else
 836                  String.Single if terminator == r"'" else String.Other)
 837         host_char = r'"' if host_double else r"'"
 838         host_quantifier = r'{3,}' if host_triple else r''
 839         host_token = String.Double if host_double else String.Single
 840         escaped_quotes = (r'+|%s(?!%s{2})' % (host_char, host_char)
 841                           if host_triple else r'')
 842         return [
 843             (r'%s%s' % (host_char, host_quantifier), host_token, '#pop:3'),
 844             (r'%s%s' % (r'' if token is String.Other else r'\\?', terminator),
 845              token, '#pop'),
 846             include('s/verbatim'),
 847             include('s/entity'),
 848             (r'\{([^}<\\%s]|<(?!<)|\\%s%s|%s|\\.)*\}' %
 849              (host_char, host_char, escaped_quotes, _escape), String.Interpol),
 850             (r'([^\s"\'<%s{}\\&])+' % (r'>' if token is String.Other else r''),
 851              token),
 852             include('s/escape'),
 853             (r'["\'\s&{<}\\]', token)
 854         ]
 855
 856     tokens = {
 857         'root': [
 858             (u'\ufeff', Text),
 859             (r'\{', Punctuation, 'object-body'),
 860             (r';+', Punctuation),
 861             (r'(?=(argcount|break|case|catch|continue|default|definingobj|'
 862              r'delegated|do|else|for|foreach|finally|goto|if|inherited|'
 863              r'invokee|local|nil|new|operator|replaced|return|self|switch|'
 864              r'targetobj|targetprop|throw|true|try|while)\b)', Text, 'block'),
 865             (r'(%s)(%s*)(\()' % (_name, _ws),
 866              bygroups(Name.Function, using(this, state='whitespace'),
 867                       Punctuation),
 868              ('block?/root', 'more/parameters', 'main/parameters')),
 869             include('whitespace'),
 870             (r'\++', Punctuation),
 871             (r'[^\s!"%-(*->@-_a-z{-~]+', Error),  # Averts an infinite loop
 872             (r'(?!\Z)', Text, 'main/root')
 873         ],
 874         'main/root': [
 875             include('main/basic'),
 876             default(('#pop', 'object-body/no-braces', 'classes', 'class'))
 877         ],
 878         'object-body/no-braces': [
 879             (r';', Punctuation, '#pop'),
 880             (r'\{', Punctuation, ('#pop', 'object-body')),
 881             include('object-body')
 882         ],
 883         'object-body': [
 884             (r';', Punctuation),
 885             (r'\{', Punctuation, '#push'),
 886             (r'\}', Punctuation, '#pop'),
 887             (r':', Punctuation, ('classes', 'class')),
 888             (r'(%s?)(%s*)(\()' % (_name, _ws),
 889              bygroups(Name.Function, using(this, state='whitespace'),
 890                       Punctuation),
 891              ('block?', 'more/parameters', 'main/parameters')),
 892             (r'(%s)(%s*)(\{)' % (_name, _ws),
 893              bygroups(Name.Function, using(this, state='whitespace'),
 894                       Punctuation), 'block'),
 895             (r'(%s)(%s*)(:)' % (_name, _ws),
 896              bygroups(Name.Variable, using(this, state='whitespace'),
 897                       Punctuation),
 898              ('object-body/no-braces', 'classes', 'class')),
 899             include('whitespace'),
 900             (r'->|%s' % _operator, Punctuation, 'main'),
 901             default('main/object-body')
 902         ],
 903         'main/object-body': [
 904             include('main/basic'),
 905             (r'(%s)(%s*)(=?)' % (_name, _ws),
 906              bygroups(Name.Variable, using(this, state='whitespace'),
 907                       Punctuation), ('#pop', 'more', 'main')),
 908             default('#pop:2')
 909         ],
 910         'block?/root': [
 911             (r'\{', Punctuation, ('#pop', 'block')),
 912             include('whitespace'),
 913             (r'(?=[[\'"<(:])', Text,  # It might be a VerbRule macro.
 914              ('#pop', 'object-body/no-braces', 'grammar', 'grammar-rules')),
 915             # It might be a macro like DefineAction.
 916             default(('#pop', 'object-body/no-braces'))
 917         ],
 918         'block?': [
 919             (r'\{', Punctuation, ('#pop', 'block')),
 920             include('whitespace'),
 921             default('#pop')
 922         ],
 923         'block/basic': [
 924             (r'[;:]+', Punctuation),
 925             (r'\{', Punctuation, '#push'),
 926             (r'\}', Punctuation, '#pop'),
 927             (r'default\b', Keyword.Reserved),
 928             (r'(%s)(%s*)(:)' % (_name, _ws),
 929              bygroups(Name.Label, using(this, state='whitespace'),
 930                       Punctuation)),
 931             include('whitespace')
 932         ],
 933         'block': [
 934             include('block/basic'),
 935             (r'(?!\Z)', Text, ('more', 'main'))
 936         ],
 937         'block/embed': [
 938             (r'>>', String.Interpol, '#pop'),
 939             include('block/basic'),
 940             (r'(?!\Z)', Text, ('more/embed', 'main'))
 941         ],
 942         'main/basic': [
 943             include('whitespace'),
 944             (r'\(', Punctuation, ('#pop', 'more', 'main')),
 945             (r'\[', Punctuation, ('#pop', 'more/list', 'main')),
 946             (r'\{', Punctuation, ('#pop', 'more/inner', 'main/inner',
 947                                   'more/parameters', 'main/parameters')),
 948             (r'\*|\.{3}', Punctuation, '#pop'),
 949             (r'(?i)0x[\da-f]+', Number.Hex, '#pop'),
 950             (r'(\d+\.(?!\.)\d*|\.\d+)([eE][-+]?\d+)?|\d+[eE][-+]?\d+',
 951              Number.Float, '#pop'),
 952             (r'0[0-7]+', Number.Oct, '#pop'),
 953             (r'\d+', Number.Integer, '#pop'),
 954             (r'"""', String.Double, ('#pop', 'tdqs')),
 955             (r"'''", String.Single, ('#pop', 'tsqs')),
 956             (r'"', String.Double, ('#pop', 'dqs')),
 957             (r"'", String.Single, ('#pop', 'sqs')),
 958             (r'R"""', String.Regex, ('#pop', 'tdqr')),
 959             (r"R'''", String.Regex, ('#pop', 'tsqr')),
 960             (r'R"', String.Regex, ('#pop', 'dqr')),
 961             (r"R'", String.Regex, ('#pop', 'sqr')),
 962             # Two-token keywords
 963             (r'(extern)(%s+)(object\b)' % _ws,
 964              bygroups(Keyword.Reserved, using(this, state='whitespace'),
 965                       Keyword.Reserved)),
 966             (r'(function|method)(%s*)(\()' % _ws,
 967              bygroups(Keyword.Reserved, using(this, state='whitespace'),
 968                       Punctuation),
 969              ('#pop', 'block?', 'more/parameters', 'main/parameters')),
 970             (r'(modify)(%s+)(grammar\b)' % _ws,
 971              bygroups(Keyword.Reserved, using(this, state='whitespace'),
 972                       Keyword.Reserved),
 973              ('#pop', 'object-body/no-braces', ':', 'grammar')),
 974             (r'(new)(%s+(?=(?:function|method)\b))' % _ws,
 975              bygroups(Keyword.Reserved, using(this, state='whitespace'))),
 976             (r'(object)(%s+)(template\b)' % _ws,
 977              bygroups(Keyword.Reserved, using(this, state='whitespace'),
 978                       Keyword.Reserved), ('#pop', 'template')),
 979             (r'(string)(%s+)(template\b)' % _ws,
 980              bygroups(Keyword, using(this, state='whitespace'),
 981                       Keyword.Reserved), ('#pop', 'function-name')),
 982             # Keywords
 983             (r'(argcount|definingobj|invokee|replaced|targetobj|targetprop)\b',
 984              Name.Builtin, '#pop'),
 985             (r'(break|continue|goto)\b', Keyword.Reserved, ('#pop', 'label')),
 986             (r'(case|extern|if|intrinsic|return|static|while)\b',
 987              Keyword.Reserved),
 988             (r'catch\b', Keyword.Reserved, ('#pop', 'catch')),
 989             (r'class\b', Keyword.Reserved,
 990              ('#pop', 'object-body/no-braces', 'class')),
 991             (r'(default|do|else|finally|try)\b', Keyword.Reserved, '#pop'),
 992             (r'(dictionary|property)\b', Keyword.Reserved,
 993              ('#pop', 'constants')),
 994             (r'enum\b', Keyword.Reserved, ('#pop', 'enum')),
 995             (r'export\b', Keyword.Reserved, ('#pop', 'main')),
 996             (r'(for|foreach)\b', Keyword.Reserved,
 997              ('#pop', 'more/inner', 'main/inner')),
 998             (r'(function|method)\b', Keyword.Reserved,
 999              ('#pop', 'block?', 'function-name')),
1000             (r'grammar\b', Keyword.Reserved,
1001              ('#pop', 'object-body/no-braces', 'grammar')),
1002             (r'inherited\b', Keyword.Reserved, ('#pop', 'inherited')),
1003             (r'local\b', Keyword.Reserved,
1004              ('#pop', 'more/local', 'main/local')),
1005             (r'(modify|replace|switch|throw|transient)\b', Keyword.Reserved,
1006              '#pop'),
1007             (r'new\b', Keyword.Reserved, ('#pop', 'class')),
1008             (r'(nil|true)\b', Keyword.Constant, '#pop'),
1009             (r'object\b', Keyword.Reserved, ('#pop', 'object-body/no-braces')),
1010             (r'operator\b', Keyword.Reserved, ('#pop', 'operator')),
1011             (r'propertyset\b', Keyword.Reserved,
1012              ('#pop', 'propertyset', 'main')),
1013             (r'self\b', Name.Builtin.Pseudo, '#pop'),
1014             (r'template\b', Keyword.Reserved, ('#pop', 'template')),
1015             # Operators
1016             (r'(__objref|defined)(%s*)(\()' % _ws,
1017              bygroups(Operator.Word, using(this, state='whitespace'),
1018                       Operator), ('#pop', 'more/__objref', 'main')),
1019             (r'delegated\b', Operator.Word),
1020             # Compiler-defined macros and built-in properties
1021             (r'(__DATE__|__DEBUG|__LINE__|__FILE__|'
1022              r'__TADS_MACRO_FORMAT_VERSION|__TADS_SYS_\w*|__TADS_SYSTEM_NAME|'
1023              r'__TADS_VERSION_MAJOR|__TADS_VERSION_MINOR|__TADS3|__TIME__|'
1024              r'construct|finalize|grammarInfo|grammarTag|lexicalParent|'
1025              r'miscVocab|sourceTextGroup|sourceTextGroupName|'
1026              r'sourceTextGroupOrder|sourceTextOrder)\b', Name.Builtin, '#pop')
1027         ],
1028         'main': [
1029             include('main/basic'),
1030             (_name, Name, '#pop'),
1031             default('#pop')
1032         ],
1033         'more/basic': [
1034             (r'\(', Punctuation, ('more/list', 'main')),
1035             (r'\[', Punctuation, ('more', 'main')),
1036             (r'\.{3}', Punctuation),
1037             (r'->|\.\.', Punctuation, 'main'),
1038             (r'(?=;)|[:)\]]', Punctuation, '#pop'),
1039             include('whitespace'),
1040             (_operator, Operator, 'main'),
1041             (r'\?', Operator, ('main', 'more/conditional', 'main')),
1042             (r'(is|not)(%s+)(in\b)' % _ws,
1043              bygroups(Operator.Word, using(this, state='whitespace'),
1044                       Operator.Word)),
1045             (r'[^\s!"%-_a-z{-~]+', Error)  # Averts an infinite loop
1046         ],
1047         'more': [
1048             include('more/basic'),
1049             default('#pop')
1050         ],
1051         # Then expression (conditional operator)
1052         'more/conditional': [
1053             (r':(?!:)', Operator, '#pop'),
1054             include('more')
1055         ],
1056         # Embedded expressions
1057         'more/embed': [
1058             (r'>>', String.Interpol, '#pop:2'),
1059             include('more')
1060         ],
1061         # For/foreach loop initializer or short-form anonymous function
1062         'main/inner': [
1063             (r'\(', Punctuation, ('#pop', 'more/inner', 'main/inner')),
1064             (r'local\b', Keyword.Reserved, ('#pop', 'main/local')),
1065             include('main')
1066         ],
1067         'more/inner': [
1068             (r'\}', Punctuation, '#pop'),
1069             (r',', Punctuation, 'main/inner'),
1070             (r'(in|step)\b', Keyword, 'main/inner'),
1071             include('more')
1072         ],
1073         # Local
1074         'main/local': [
1075             (_name, Name.Variable, '#pop'),
1076             include('whitespace')
1077         ],
1078         'more/local': [
1079             (r',', Punctuation, 'main/local'),
1080             include('more')
1081         ],
1082         # List
1083         'more/list': [
1084             (r'[,:]', Punctuation, 'main'),
1085             include('more')
1086         ],
1087         # Parameter list
1088         'main/parameters': [
1089             (r'(%s)(%s*)(?=:)' % (_name, _ws),
1090              bygroups(Name.Variable, using(this, state='whitespace')), '#pop'),
1091             (r'(%s)(%s+)(%s)' % (_name, _ws, _name),
1092              bygroups(Name.Class, using(this, state='whitespace'),
1093                       Name.Variable), '#pop'),
1094             (r'\[+', Punctuation),
1095             include('main/basic'),
1096             (_name, Name.Variable, '#pop'),
1097             default('#pop')
1098         ],
1099         'more/parameters': [
1100             (r'(:)(%s*(?=[?=,:)]))' % _ws,
1101              bygroups(Punctuation, using(this, state='whitespace'))),
1102             (r'[?\]]+', Punctuation),
1103             (r'[:)]', Punctuation, ('#pop', 'multimethod?')),
1104             (r',', Punctuation, 'main/parameters'),
1105             (r'=', Punctuation, ('more/parameter', 'main')),
1106             include('more')
1107         ],
1108         'more/parameter': [
1109             (r'(?=[,)])', Text, '#pop'),
1110             include('more')
1111         ],
1112         'multimethod?': [
1113             (r'multimethod\b', Keyword, '#pop'),
1114             include('whitespace'),
1115             default('#pop')
1116         ],
1117
1118         # Statements and expressions
1119         'more/__objref': [
1120             (r',', Punctuation, 'mode'),
1121             (r'\)', Operator, '#pop'),
1122             include('more')
1123         ],
1124         'mode': [
1125             (r'(error|warn)\b', Keyword, '#pop'),
1126             include('whitespace')
1127         ],
1128         'catch': [
1129             (r'\(+', Punctuation),
1130             (_name, Name.Exception, ('#pop', 'variables')),
1131             include('whitespace')
1132         ],
1133         'enum': [
1134             include('whitespace'),
1135             (r'token\b', Keyword, ('#pop', 'constants')),
1136             default(('#pop', 'constants'))
1137         ],
1138         'grammar': [
1139             (r'\)+', Punctuation),
1140             (r'\(', Punctuation, 'grammar-tag'),
1141             (r':', Punctuation, 'grammar-rules'),
1142             (_name, Name.Class),
1143             include('whitespace')
1144         ],
1145         'grammar-tag': [
1146             include('whitespace'),
1147             (r'"""([^\\"<]|""?(?!")|\\"+|\\.|<(?!<))+("{3,}|<<)|'
1148              r'R"""([^\\"]|""?(?!")|\\"+|\\.)+"{3,}|'
1149              r"'''([^\\'<]|''?(?!')|\\'+|\\.|<(?!<))+('{3,}|<<)|"
1150              r"R'''([^\\']|''?(?!')|\\'+|\\.)+'{3,}|"
1151              r'"([^\\"<]|\\.|<(?!<))+("|<<)|R"([^\\"]|\\.)+"|'
1152              r"'([^\\'<]|\\.|<(?!<))+('|<<)|R'([^\\']|\\.)+'|"
1153              r"([^)\s\\/]|/(?![/*]))+|\)", String.Other, '#pop')
1154         ],
1155         'grammar-rules': [
1156             include('string'),
1157             include('whitespace'),
1158             (r'(\[)(%s*)(badness)' % _ws,
1159              bygroups(Punctuation, using(this, state='whitespace'), Keyword),
1160              'main'),
1161             (r'->|%s|[()]' % _operator, Punctuation),
1162             (_name, Name.Constant),
1163             default('#pop:2')
1164         ],
1165         ':': [
1166             (r':', Punctuation, '#pop')
1167         ],
1168         'function-name': [
1169             (r'(<<([^>]|>>>|>(?!>))*>>)+', String.Interpol),
1170             (r'(?=%s?%s*[({])' % (_name, _ws), Text, '#pop'),
1171             (_name, Name.Function, '#pop'),
1172             include('whitespace')
1173         ],
1174         'inherited': [
1175             (r'<', Punctuation, ('#pop', 'classes', 'class')),
1176             include('whitespace'),
1177             (_name, Name.Class, '#pop'),
1178             default('#pop')
1179         ],
1180         'operator': [
1181             (r'negate\b', Operator.Word, '#pop'),
1182             include('whitespace'),
1183             (_operator, Operator),
1184             default('#pop')
1185         ],
1186         'propertyset': [
1187             (r'\(', Punctuation, ('more/parameters', 'main/parameters')),
1188             (r'\{', Punctuation, ('#pop', 'object-body')),
1189             include('whitespace')
1190         ],
1191         'template': [
1192             (r'(?=;)', Text, '#pop'),
1193             include('string'),
1194             (r'inherited\b', Keyword.Reserved),
1195             include('whitespace'),
1196             (r'->|\?|%s' % _operator, Punctuation),
1197             (_name, Name.Variable)
1198         ],
1199
1200         # Identifiers
1201         'class': [
1202             (r'\*|\.{3}', Punctuation, '#pop'),
1203             (r'object\b', Keyword.Reserved, '#pop'),
1204             (r'transient\b', Keyword.Reserved),
1205             (_name, Name.Class, '#pop'),
1206             include('whitespace'),
1207             default('#pop')
1208         ],
1209         'classes': [
1210             (r'[:,]', Punctuation, 'class'),
1211             include('whitespace'),
1212             (r'>', Punctuation, '#pop'),
1213             default('#pop')
1214         ],
1215         'constants': [
1216             (r',+', Punctuation),
1217             (r';', Punctuation, '#pop'),
1218             (r'property\b', Keyword.Reserved),
1219             (_name, Name.Constant),
1220             include('whitespace')
1221         ],
1222         'label': [
1223             (_name, Name.Label, '#pop'),
1224             include('whitespace'),
1225             default('#pop')
1226         ],
1227         'variables': [
1228             (r',+', Punctuation),
1229             (r'\)', Punctuation, '#pop'),
1230             include('whitespace'),
1231             (_name, Name.Variable)
1232         ],
1233
1234         # Whitespace and comments
1235         'whitespace': [
1236             (r'^%s*#(%s|[^\n]|(?<=\\)\n)*\n?' % (_ws_pp, _comment_multiline),
1237              Comment.Preproc),
1238             (_comment_single, Comment.Single),
1239             (_comment_multiline, Comment.Multiline),
1240             (r'\\+\n+%s*#?|\n+|([^\S\n]|\\)+' % _ws_pp, Text)
1241         ],
1242
1243         # Strings
1244         'string': [
1245             (r'"""', String.Double, 'tdqs'),
1246             (r"'''", String.Single, 'tsqs'),
1247             (r'"', String.Double, 'dqs'),
1248             (r"'", String.Single, 'sqs')
1249         ],
1250         's/escape': [
1251             (r'\{\{|\}\}|%s' % _escape, String.Escape)
1252         ],
1253         's/verbatim': [
1254             (r'<<\s*(as\s+decreasingly\s+likely\s+outcomes|cycling|else|end|'
1255              r'first\s+time|one\s+of|only|or|otherwise|'
1256              r'(sticky|(then\s+)?(purely\s+)?at)\s+random|stopping|'
1257              r'(then\s+)?(half\s+)?shuffled|\|\|)\s*>>', String.Interpol),
1258             (r'<<(%%(_(%s|\\?.)|[\-+ ,#]|\[\d*\]?)*\d*\.?\d*(%s|\\?.)|'
1259              r'\s*((else|otherwise)\s+)?(if|unless)\b)?' % (_escape, _escape),
1260              String.Interpol, ('block/embed', 'more/embed', 'main'))
1261         ],
1262         's/entity': [
1263             (r'(?i)&(#(x[\da-f]+|\d+)|[a-z][\da-z]*);?', Name.Entity)
1264         ],
1265         'tdqs': _make_string_state(True, True),
1266         'tsqs': _make_string_state(True, False),
1267         'dqs': _make_string_state(False, True),
1268         'sqs': _make_string_state(False, False),
1269         'tdqs/listing': _make_string_state(True, True, 'listing'),
1270         'tsqs/listing': _make_string_state(True, False, 'listing'),
1271         'dqs/listing': _make_string_state(False, True, 'listing'),
1272         'sqs/listing': _make_string_state(False, False, 'listing'),
1273         'tdqs/xmp': _make_string_state(True, True, 'xmp'),
1274         'tsqs/xmp': _make_string_state(True, False, 'xmp'),
1275         'dqs/xmp': _make_string_state(False, True, 'xmp'),
1276         'sqs/xmp': _make_string_state(False, False, 'xmp'),
1277
1278         # Tags
1279         'tdqt': _make_tag_state(True, True),
1280         'tsqt': _make_tag_state(True, False),
1281         'dqt': _make_tag_state(False, True),
1282         'sqt': _make_tag_state(False, False),
1283         'dqs/tdqt': _make_attribute_value_state(r'"', True, True),
1284         'dqs/tsqt': _make_attribute_value_state(r'"', True, False),
1285         'dqs/dqt': _make_attribute_value_state(r'"', False, True),
1286         'dqs/sqt': _make_attribute_value_state(r'"', False, False),
1287         'sqs/tdqt': _make_attribute_value_state(r"'", True, True),
1288         'sqs/tsqt': _make_attribute_value_state(r"'", True, False),
1289         'sqs/dqt': _make_attribute_value_state(r"'", False, True),
1290         'sqs/sqt': _make_attribute_value_state(r"'", False, False),
1291         'uqs/tdqt': _make_attribute_value_state(_no_quote, True, True),
1292         'uqs/tsqt': _make_attribute_value_state(_no_quote, True, False),
1293         'uqs/dqt': _make_attribute_value_state(_no_quote, False, True),
1294         'uqs/sqt': _make_attribute_value_state(_no_quote, False, False),
1295
1296         # Regular expressions
1297         'tdqr': [
1298             (r'[^\\"]+', String.Regex),
1299             (r'\\"*', String.Regex),
1300             (r'"{3,}', String.Regex, '#pop'),
1301             (r'"', String.Regex)
1302         ],
1303         'tsqr': [
1304             (r"[^\\']+", String.Regex),
1305             (r"\\'*", String.Regex),
1306             (r"'{3,}", String.Regex, '#pop'),
1307             (r"'", String.Regex)
1308         ],
1309         'dqr': [
1310             (r'[^\\"]+', String.Regex),
1311             (r'\\"?', String.Regex),
1312             (r'"', String.Regex, '#pop')
1313         ],
1314         'sqr': [
1315             (r"[^\\']+", String.Regex),
1316             (r"\\'?", String.Regex),
1317             (r"'", String.Regex, '#pop')
1318         ]
1319     }
1320
1321     def get_tokens_unprocessed(self, text, **kwargs):
1322         pp = r'^%s*#%s*' % (self._ws_pp, self._ws_pp)
1323         if_false_level = 0
1324         for index, token, value in (
1325             RegexLexer.get_tokens_unprocessed(self, text, **kwargs)):
1326             if if_false_level == 0:  # Not in a false #if
1327                 if (token is Comment.Preproc and
1328                     re.match(r'%sif%s+(0|nil)%s*$\n?' %
1329                              (pp, self._ws_pp, self._ws_pp), value)):
1330                     if_false_level = 1
1331             else:  # In a false #if
1332                 if token is Comment.Preproc:
1333                     if (if_false_level == 1 and
1334                           re.match(r'%sel(if|se)\b' % pp, value)):
1335                         if_false_level = 0
1336                     elif re.match(r'%sif' % pp, value):
1337                         if_false_level += 1
1338                     elif re.match(r'%sendif\b' % pp, value):
1339                         if_false_level -= 1
1340                 else:
1341                     token = Comment
1342             yield index, token, value