File Explorer

1# Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved.2# Licensed to PSF under a Contributor Agreement.3 4"""This module defines the data structures used to represent a grammar.5 6These are a bit arcane because they are derived from the data7structures used by Python's 'pgen' parser generator.8 9There's also a table here mapping operators to their names in the10token module; the Python tokenize module reports all operators as the11fallback token code OP, but the parser needs the actual token code.12 13"""14 15# Python imports16import pickle17 18# Local imports19from . import token20 21 22class Grammar(object):23    """Pgen parsing tables conversion class.24 25    Once initialized, this class supplies the grammar tables for the26    parsing engine implemented by parse.py.  The parsing engine27    accesses the instance variables directly.  The class here does not28    provide initialization of the tables; several subclasses exist to29    do this (see the conv and pgen modules).30 31    The load() method reads the tables from a pickle file, which is32    much faster than the other ways offered by subclasses.  The pickle33    file is written by calling dump() (after loading the grammar34    tables using a subclass).  The report() method prints a readable35    representation of the tables to stdout, for debugging.36 37    The instance variables are as follows:38 39    symbol2number -- a dict mapping symbol names to numbers.  Symbol40                     numbers are always 256 or higher, to distinguish41                     them from token numbers, which are between 0 and42                     255 (inclusive).43 44    number2symbol -- a dict mapping numbers to symbol names;45                     these two are each other's inverse.46 47    states        -- a list of DFAs, where each DFA is a list of48                     states, each state is a list of arcs, and each49                     arc is a (i, j) pair where i is a label and j is50                     a state number.  The DFA number is the index into51                     this list.  (This name is slightly confusing.)52                     Final states are represented by a special arc of53                     the form (0, j) where j is its own state number.54 55    dfas          -- a dict mapping symbol numbers to (DFA, first)56                     pairs, where DFA is an item from the states list57                     above, and first is a set of tokens that can58                     begin this grammar rule (represented by a dict59                     whose values are always 1).60 61    labels        -- a list of (x, y) pairs where x is either a token62                     number or a symbol number, and y is either None63                     or a string; the strings are keywords.  The label64                     number is the index in this list; label numbers65                     are used to mark state transitions (arcs) in the66                     DFAs.67 68    start         -- the number of the grammar's start symbol.69 70    keywords      -- a dict mapping keyword strings to arc labels.71 72    tokens        -- a dict mapping token numbers to arc labels.73 74    """75 76    def __init__(self):77        self.symbol2number = {}78        self.number2symbol = {}79        self.states = []80        self.dfas = {}81        self.labels = [(0, "EMPTY")]82        self.keywords = {}83        self.tokens = {}84        self.symbol2label = {}85        self.start = 25686 87    def dump(self, filename):88        """Dump the grammar tables to a pickle file."""89        with open(filename, "wb") as f:90            pickle.dump(self.__dict__, f, pickle.HIGHEST_PROTOCOL)91 92    def load(self, filename):93        """Load the grammar tables from a pickle file."""94        with open(filename, "rb") as f:95            d = pickle.load(f)96        self.__dict__.update(d)97 98    def loads(self, pkl):99        """Load the grammar tables from a pickle bytes object."""100        self.__dict__.update(pickle.loads(pkl))101 102    def copy(self):103        """104        Copy the grammar.105        """106        new = self.__class__()107        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",108                          "tokens", "symbol2label"):109            setattr(new, dict_attr, getattr(self, dict_attr).copy())110        new.labels = self.labels[:]111        new.states = self.states[:]112        new.start = self.start113        return new114 115    def report(self):116        """Dump the grammar tables to standard output, for debugging."""117        from pprint import pprint118        print("s2n")119        pprint(self.symbol2number)120        print("n2s")121        pprint(self.number2symbol)122        print("states")123        pprint(self.states)124        print("dfas")125        pprint(self.dfas)126        print("labels")127        pprint(self.labels)128        print("start", self.start)129 130 131# Map from operator to number (since tokenize doesn't do this)132 133opmap_raw = """134( LPAR135) RPAR136[ LSQB137] RSQB138: COLON139, COMMA140; SEMI141+ PLUS142- MINUS143* STAR144/ SLASH145| VBAR146& AMPER147< LESS148> GREATER149= EQUAL150. DOT151% PERCENT152` BACKQUOTE153{ LBRACE154} RBRACE155@ AT156@= ATEQUAL157== EQEQUAL158!= NOTEQUAL159<> NOTEQUAL160<= LESSEQUAL161>= GREATEREQUAL162~ TILDE163^ CIRCUMFLEX164<< LEFTSHIFT165>> RIGHTSHIFT166** DOUBLESTAR167+= PLUSEQUAL168-= MINEQUAL169*= STAREQUAL170/= SLASHEQUAL171%= PERCENTEQUAL172&= AMPEREQUAL173|= VBAREQUAL174^= CIRCUMFLEXEQUAL175<<= LEFTSHIFTEQUAL176>>= RIGHTSHIFTEQUAL177**= DOUBLESTAREQUAL178// DOUBLESLASH179//= DOUBLESLASHEQUAL180-> RARROW181:= COLONEQUAL182"""183 184opmap = {}185for line in opmap_raw.splitlines():186    if line:187        op, name = line.split()188        opmap[op] = getattr(token, name)189