"""A flow graph representation for Python bytecode""" import dis import types import sys from compiler import misc from compiler.consts \ import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS class FlowGraph: def __init__(self): self.current = self.entry = Block() self.exit = Block("exit") self.blocks = misc.Set() self.blocks.add(self.entry) self.blocks.add(self.exit) def startBlock(self, block): if self._debug: if self.current: print "end", repr(self.current) print " next", self.current.next print " prev", self.current.prev print " ", self.current.get_children() print repr(block) self.current = block def nextBlock(self, block=None): # XXX think we need to specify when there is implicit transfer # from one block to the next. might be better to represent this # with explicit JUMP_ABSOLUTE instructions that are optimized # out when they are unnecessary. # # I think this strategy works: each block has a child # designated as "next" which is returned as the last of the # children. because the nodes in a graph are emitted in # reverse post order, the "next" block will always be emitted # immediately after its parent. # Worry: maintaining this invariant could be tricky if block is None: block = self.newBlock() # Note: If the current block ends with an unconditional control # transfer, then it is techically incorrect to add an implicit # transfer to the block graph. Doing so results in code generation # for unreachable blocks. That doesn't appear to be very common # with Python code and since the built-in compiler doesn't optimize # it out we don't either. self.current.addNext(block) self.startBlock(block) def newBlock(self): b = Block() self.blocks.add(b) return b def startExitBlock(self): self.startBlock(self.exit) _debug = 0 def _enable_debug(self): self._debug = 1 def _disable_debug(self): self._debug = 0 def emit(self, *inst): if self._debug: print "\t", inst if len(inst) == 2 and isinstance(inst[1], Block): self.current.addOutEdge(inst[1]) self.current.emit(inst) def getBlocksInOrder(self): """Return the blocks in reverse postorder i.e. each node appears before all of its successors """ order = order_blocks(self.entry, self.exit) return order def getBlocks(self): return self.blocks.elements() def getRoot(self): """Return nodes appropriate for use with dominator""" return self.entry def getContainedGraphs(self): l = [] for b in self.getBlocks(): l.extend(b.getContainedGraphs()) return l def order_blocks(start_block, exit_block): """Order blocks so that they are emitted in the right order""" # Rules: # - when a block has a next block, the next block must be emitted just after # - when a block has followers (relative jumps), it must be emitted before # them # - all reachable blocks must be emitted order = [] # Find all the blocks to be emitted. remaining = set() todo = [start_block] while todo: b = todo.pop() if b in remaining: continue remaining.add(b) for c in b.get_children(): if c not in remaining: todo.append(c) # A block is dominated by another block if that block must be emitted # before it. dominators = {} for b in remaining: if __debug__ and b.next: assert b is b.next[0].prev[0], (b, b.next) # Make sure every block appears in dominators, even if no # other block must precede it. dominators.setdefault(b, set()) # preceeding blocks dominate following blocks for c in b.get_followers(): while 1: dominators.setdefault(c, set()).add(b) # Any block that has a next pointer leading to c is also # dominated because the whole chain will be emitted at once. # Walk backwards and add them all. if c.prev and c.prev[0] is not b: c = c.prev[0] else: break def find_next(): # Find a block that can be emitted next. for b in remaining: for c in dominators[b]: if c in remaining: break # can't emit yet, dominated by a remaining block else: return b assert 0, 'circular dependency, cannot find next block' b = start_block while 1: order.append(b) remaining.discard(b) if b.next: b = b.next[0] continue elif b is not exit_block and not b.has_unconditional_transfer(): order.append(exit_block) if not remaining: break b = find_next() return order class Block: _count = 0 def __init__(self, label=''): self.insts = [] self.outEdges = set() self.label = label self.bid = Block._count self.next = [] self.prev = [] Block._count = Block._count + 1 def __repr__(self): if self.label: return "<block %s id=%d>" % (self.label, self.bid) else: return "<block id=%d>" % (self.bid) def __str__(self): insts = map(str, self.insts) return "<block %s %d:\n%s>" % (self.label, self.bid, '\n'.join(insts)) def emit(self, inst): op = inst[0] self.insts.append(inst) def getInstructions(self): return self.insts def addOutEdge(self, block): self.outEdges.add(block) def addNext(self, block): self.next.append(block) assert len(self.next) == 1, map(str, self.next) block.prev.append(self) assert len(block.prev) == 1, map(str, block.prev) _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP', ) def has_unconditional_transfer(self): """Returns True if there is an unconditional transfer to an other block at the end of this block. This means there is no risk for the bytecode executer to go past this block's bytecode.""" try: op, arg = self.insts[-1] except (IndexError, ValueError): return return op in self._uncond_transfer def get_children(self): return list(self.outEdges) + self.next def get_followers(self): """Get the whole list of followers, including the next block.""" followers = set(self.next) # Blocks that must be emitted *after* this one, because of # bytecode offsets (e.g. relative jumps) pointing to them. for inst in self.insts: if inst[0] in PyFlowGraph.hasjrel: followers.add(inst[1]) return followers def getContainedGraphs(self): """Return all graphs contained within this block. For example, a MAKE_FUNCTION block will contain a reference to the graph for the function body. """ contained = [] for inst in self.insts: if len(inst) == 1: continue op = inst[1] if hasattr(op, 'graph'): contained.append(op.graph) return contained # flags for code objects # the FlowGraph is transformed in place; it exists in one of these states RAW = "RAW" FLAT = "FLAT" CONV = "CONV" DONE = "DONE" class PyFlowGraph(FlowGraph): super_init = FlowGraph.__init__ def __init__(self, name, filename, args=(), optimized=0, klass=None): self.super_init() self.name = name self.filename = filename self.docstring = None self.args = args # XXX self.argcount = getArgCount(args) self.klass = klass if optimized: self.flags = CO_OPTIMIZED | CO_NEWLOCALS else: self.flags = 0 self.consts = [] self.names = [] # Free variables found by the symbol table scan, including # variables used only in nested scopes, are included here. self.freevars = [] self.cellvars = [] # The closure list is used to track the order of cell # variables and free variables in the resulting code object. # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both # kinds of variables. self.closure = [] self.varnames = list(args) or [] for i in range(len(self.varnames)): var = self.varnames[i] if isinstance(var, TupleArg): self.varnames[i] = var.getName() self.stage = RAW def setDocstring(self, doc): self.docstring = doc def setFlag(self, flag): self.flags = self.flags | flag if flag == CO_VARARGS: self.argcount = self.argcount - 1 def checkFlag(self, flag): if self.flags & flag: return 1 def setFreeVars(self, names): self.freevars = list(names) def setCellVars(self, names): self.cellvars = names def getCode(self): """Get a Python code object""" assert self.stage == RAW self.computeStackDepth() self.flattenGraph() assert self.stage == FLAT self.convertArgs() assert self.stage == CONV self.makeByteCode() assert self.stage == DONE return self.newCodeObject() def dump(self, io=None): if io: save = sys.stdout sys.stdout = io pc = 0 for t in self.insts: opname = t[0] if opname == "SET_LINENO": print if len(t) == 1: print "\t", "%3d" % pc, opname pc = pc + 1 else: print "\t", "%3d" % pc, opname, t[1] pc = pc + 3 if io: sys.stdout = save def computeStackDepth(self): """Compute the max stack depth. Approach is to compute the stack effect of each basic block. Then find the path through the code with the largest total effect. """ depth = {} exit = None for b in self.getBlocks(): depth[b] = findDepth(b.getInstructions()) seen = {} def max_depth(b, d): if b in seen: return d seen[b] = 1 d = d + depth[b] children = b.get_children() if children: return max([max_depth(c, d) for c in children]) else: if not b.label == "exit": return max_depth(self.exit, d) else: return d self.stacksize = max_depth(self.entry, 0) def flattenGraph(self): """Arrange the blocks in order and resolve jumps""" assert self.stage == RAW self.insts = insts = [] pc = 0 begin = {} end = {} for b in self.getBlocksInOrder(): begin[b] = pc for inst in b.getInstructions(): insts.append(inst) if len(inst) == 1: pc = pc + 1 elif inst[0] != "SET_LINENO": # arg takes 2 bytes pc = pc + 3 end[b] = pc pc = 0 for i in range(len(insts)): inst = insts[i] if len(inst) == 1: pc = pc + 1 elif inst[0] != "SET_LINENO": pc = pc + 3 opname = inst[0] if opname in self.hasjrel: oparg = inst[1] offset = begin[oparg] - pc insts[i] = opname, offset elif opname in self.hasjabs: insts[i] = opname, begin[inst[1]] self.stage = FLAT hasjrel = set() for i in dis.hasjrel: hasjrel.add(dis.opname[i]) hasjabs = set() for i in dis.hasjabs: hasjabs.add(dis.opname[i]) def convertArgs(self): """Convert arguments from symbolic to concrete form""" assert self.stage == FLAT self.consts.insert(0, self.docstring) self.sort_cellvars() for i in range(len(self.insts)): t = self.insts[i] if len(t) == 2: opname, oparg = t conv = self._converters.get(opname, None) if conv: self.insts[i] = opname, conv(self, oparg) self.stage = CONV def sort_cellvars(self): """Sort cellvars in the order of varnames and prune from freevars. """ cells = {} for name in self.cellvars: cells[name] = 1 self.cellvars = [name for name in self.varnames if name in cells] for name in self.cellvars: del cells[name] self.cellvars = self.cellvars + cells.keys() self.closure = self.cellvars + self.freevars def _lookupName(self, name, list): """Return index of name in list, appending if necessary This routine uses a list instead of a dictionary, because a dictionary can't store two different keys if the keys have the same value but different types, e.g. 2 and 2L. The compiler must treat these two separately, so it does an explicit type comparison before comparing the values. """ t = type(name) for i in range(len(list)): if t == type(list[i]) and list[i] == name: return i end = len(list) list.append(name) return end _converters = {} def _convert_LOAD_CONST(self, arg): if hasattr(arg, 'getCode'): arg = arg.getCode() return self._lookupName(arg, self.consts) def _convert_LOAD_FAST(self, arg): self._lookupName(arg, self.names) return self._lookupName(arg, self.varnames) _convert_STORE_FAST = _convert_LOAD_FAST _convert_DELETE_FAST = _convert_LOAD_FAST def _convert_LOAD_NAME(self, arg): if self.klass is None: self._lookupName(arg, self.varnames) return self._lookupName(arg, self.names) def _convert_NAME(self, arg): if self.klass is None: self._lookupName(arg, self.varnames) return self._lookupName(arg, self.names) _convert_STORE_NAME = _convert_NAME _convert_DELETE_NAME = _convert_NAME _convert_IMPORT_NAME = _convert_NAME _convert_IMPORT_FROM = _convert_NAME _convert_STORE_ATTR = _convert_NAME _convert_LOAD_ATTR = _convert_NAME _convert_DELETE_ATTR = _convert_NAME _convert_LOAD_GLOBAL = _convert_NAME _convert_STORE_GLOBAL = _convert_NAME _convert_DELETE_GLOBAL = _convert_NAME def _convert_DEREF(self, arg): self._lookupName(arg, self.names) self._lookupName(arg, self.varnames) return self._lookupName(arg, self.closure) _convert_LOAD_DEREF = _convert_DEREF _convert_STORE_DEREF = _convert_DEREF def _convert_LOAD_CLOSURE(self, arg): self._lookupName(arg, self.varnames) return self._lookupName(arg, self.closure) _cmp = list(dis.cmp_op) def _convert_COMPARE_OP(self, arg): return self._cmp.index(arg) # similarly for other opcodes... for name, obj in locals().items(): if name[:9] == "_convert_": opname = name[9:] _converters[opname] = obj del name, obj, opname def makeByteCode(self): assert self.stage == CONV self.lnotab = lnotab = LineAddrTable() for t in self.insts: opname = t[0] if len(t) == 1: lnotab.addCode(self.opnum[opname]) else: oparg = t[1] if opname == "SET_LINENO": lnotab.nextLine(oparg) continue hi, lo = twobyte(oparg) try: lnotab.addCode(self.opnum[opname], lo, hi) except ValueError: print opname, oparg print self.opnum[opname], lo, hi raise self.stage = DONE opnum = {} for num in range(len(dis.opname)): opnum[dis.opname[num]] = num del num def newCodeObject(self): assert self.stage == DONE if (self.flags & CO_NEWLOCALS) == 0: nlocals = 0 else: nlocals = len(self.varnames) argcount = self.argcount if self.flags & CO_VARKEYWORDS: argcount = argcount - 1 return types.CodeType(argcount, nlocals, self.stacksize, self.flags, self.lnotab.getCode(), self.getConsts(), tuple(self.names), tuple(self.varnames), self.filename, self.name, self.lnotab.firstline, self.lnotab.getTable(), tuple(self.freevars), tuple(self.cellvars)) def getConsts(self): """Return a tuple for the const slot of the code object Must convert references to code (MAKE_FUNCTION) to code objects recursively. """ l = [] for elt in self.consts: if isinstance(elt, PyFlowGraph): elt = elt.getCode() l.append(elt) return tuple(l) def isJump(opname): if opname[:4] == 'JUMP': return 1 class TupleArg: """Helper for marking func defs with nested tuples in arglist""" def __init__(self, count, names): self.count = count self.names = names def __repr__(self): return "TupleArg(%s, %s)" % (self.count, self.names) def getName(self): return ".%d" % self.count def getArgCount(args): argcount = len(args) if args: for arg in args: if isinstance(arg, TupleArg): numNames = len(misc.flatten(arg.names)) argcount = argcount - numNames return argcount def twobyte(val): """Convert an int argument into high and low bytes""" assert isinstance(val, int) return divmod(val, 256) class LineAddrTable: """lnotab This class builds the lnotab, which is documented in compile.c. Here's a brief recap: For each SET_LINENO instruction after the first one, two bytes are added to lnotab. (In some cases, multiple two-byte entries are added.) The first byte is the distance in bytes between the instruction for the last SET_LINENO and the current SET_LINENO. The second byte is offset in line numbers. If either offset is greater than 255, multiple two-byte entries are added -- see compile.c for the delicate details. """ def __init__(self): self.code = [] self.codeOffset = 0 self.firstline = 0 self.lastline = 0 self.lastoff = 0 self.lnotab = [] def addCode(self, *args): for arg in args: self.code.append(chr(arg)) self.codeOffset = self.codeOffset + len(args) def nextLine(self, lineno): if self.firstline == 0: self.firstline = lineno self.lastline = lineno else: # compute deltas addr = self.codeOffset - self.lastoff line = lineno - self.lastline # Python assumes that lineno always increases with # increasing bytecode address (lnotab is unsigned char). # Depending on when SET_LINENO instructions are emitted # this is not always true. Consider the code: # a = (1, # b) # In the bytecode stream, the assignment to "a" occurs # after the loading of "b". This works with the C Python # compiler because it only generates a SET_LINENO instruction # for the assignment. if line >= 0: push = self.lnotab.append while addr > 255: push(255); push(0) addr -= 255 while line > 255: push(addr); push(255) line -= 255 addr = 0 if addr > 0 or line > 0: push(addr); push(line) self.lastline = lineno self.lastoff = self.codeOffset def getCode(self): return ''.join(self.code) def getTable(self): return ''.join(map(chr, self.lnotab)) class StackDepthTracker: # XXX 1. need to keep track of stack depth on jumps # XXX 2. at least partly as a result, this code is broken def findDepth(self, insts, debug=0): depth = 0 maxDepth = 0 for i in insts: opname = i[0] if debug: print i, delta = self.effect.get(opname, None) if delta is not None: depth = depth + delta else: # now check patterns for pat, pat_delta in self.patterns: if opname[:len(pat)] == pat: delta = pat_delta depth = depth + delta break # if we still haven't found a match if delta is None: meth = getattr(self, opname, None) if meth is not None: depth = depth + meth(i[1]) if depth > maxDepth: maxDepth = depth if debug: print depth, maxDepth return maxDepth effect = { 'POP_TOP': -1, 'DUP_TOP': 1, 'LIST_APPEND': -1, 'SET_ADD': -1, 'MAP_ADD': -2, 'SLICE+1': -1, 'SLICE+2': -1, 'SLICE+3': -2, 'STORE_SLICE+0': -1, 'STORE_SLICE+1': -2, 'STORE_SLICE+2': -2, 'STORE_SLICE+3': -3, 'DELETE_SLICE+0': -1, 'DELETE_SLICE+1': -2, 'DELETE_SLICE+2': -2, 'DELETE_SLICE+3': -3, 'STORE_SUBSCR': -3, 'DELETE_SUBSCR': -2, # PRINT_EXPR? 'PRINT_ITEM': -1, 'RETURN_VALUE': -1, 'YIELD_VALUE': -1, 'EXEC_STMT': -3, 'BUILD_CLASS': -2, 'STORE_NAME': -1, 'STORE_ATTR': -2, 'DELETE_ATTR': -1, 'STORE_GLOBAL': -1, 'BUILD_MAP': 1, 'COMPARE_OP': -1, 'STORE_FAST': -1, 'IMPORT_STAR': -1, 'IMPORT_NAME': -1, 'IMPORT_FROM': 1, 'LOAD_ATTR': 0, # unlike other loads # close enough... 'SETUP_EXCEPT': 3, 'SETUP_FINALLY': 3, 'FOR_ITER': 1, 'WITH_CLEANUP': -1, } # use pattern match patterns = [ ('BINARY_', -1), ('LOAD_', 1), ] def UNPACK_SEQUENCE(self, count): return count-1 def BUILD_TUPLE(self, count): return -count+1 def BUILD_LIST(self, count): return -count+1 def BUILD_SET(self, count): return -count+1 def CALL_FUNCTION(self, argc): hi, lo = divmod(argc, 256) return -(lo + hi * 2) def CALL_FUNCTION_VAR(self, argc): return self.CALL_FUNCTION(argc)-1 def CALL_FUNCTION_KW(self, argc): return self.CALL_FUNCTION(argc)-1 def CALL_FUNCTION_VAR_KW(self, argc): return self.CALL_FUNCTION(argc)-2 def MAKE_FUNCTION(self, argc): return -argc def MAKE_CLOSURE(self, argc): # XXX need to account for free variables too! return -argc def BUILD_SLICE(self, argc): if argc == 2: return -1 elif argc == 3: return -2 def DUP_TOPX(self, argc): return argc findDepth = StackDepthTracker().findDepth