Coverage for larch/interpreter.py: 71%

720 statements  

« prev     ^ index     » next       coverage.py v7.6.0, created at 2024-10-16 21:04 +0000

1#!/usr/bin/env python 

2""" 

3 Main Larch interpreter 

4 

5Safe(ish) evaluator of python expressions, using ast module. 

6The emphasis here is on mathematical expressions, and so 

7numpy functions are imported if available and used. 

8""" 

9import os 

10import sys 

11import types 

12import ast 

13import math 

14import numpy 

15from pathlib import Path 

16from copy import deepcopy 

17 

18from . import site_config 

19from asteval import valid_symbol_name 

20from .symboltable import SymbolTable, Group, isgroup 

21from .inputText import InputText, BLANK_TEXT 

22from .larchlib import (LarchExceptionHolder, ReturnedNone, 

23 Procedure, StdWriter) 

24from .closure import Closure 

25 

26UNSAFE_ATTRS = ('__subclasses__', '__bases__', '__globals__', '__code__', 

27 '__closure__', '__func__', '__self__', '__module__', 

28 '__dict__', '__class__', '__call__', '__get__', 

29 '__getattribute__', '__subclasshook__', '__new__', 

30 '__init__', 'func_globals', 'func_code', 'func_closure', 

31 'im_class', 'im_func', 'im_self', 'gi_code', 'gi_frame', 

32 '__asteval__', 'f_locals', '__mro__') 

33 

34 

35OPERATORS = { 

36 ast.Add: lambda a, b: a + b, 

37 ast.Sub: lambda a, b: a - b, 

38 ast.Mult: lambda a, b: a * b, 

39 ast.Div: lambda a, b: a/b, 

40 ast.FloorDiv: lambda a, b: a//b, 

41 ast.Mod: lambda a, b: a % b, 

42 ast.Pow: lambda a, b: a ** b, 

43 ast.Eq: lambda a, b: a == b, 

44 ast.Gt: lambda a, b: a > b, 

45 ast.GtE: lambda a, b: a >= b, 

46 ast.Lt: lambda a, b: a < b, 

47 ast.LtE: lambda a, b: a <= b, 

48 ast.NotEq: lambda a, b: a != b, 

49 ast.Is: lambda a, b: a is b, 

50 ast.IsNot: lambda a, b: a is not b, 

51 ast.In: lambda a, b: a in b, 

52 ast.NotIn: lambda a, b: a not in b, 

53 ast.BitAnd: lambda a, b: a & b, 

54 ast.BitOr: lambda a, b: a | b, 

55 ast.BitXor: lambda a, b: a ^ b, 

56 ast.LShift: lambda a, b: a << b, 

57 ast.RShift: lambda a, b: a >> b, 

58 ast.And: lambda a, b: a and b, 

59 ast.Or: lambda a, b: a or b, 

60 ast.Invert: lambda a: ~a, 

61 ast.Not: lambda a: not a, 

62 ast.UAdd: lambda a: +a, 

63 ast.USub: lambda a: -a} 

64 

65PYTHON_RESERVED_WORDS = ('and', 'as', 'assert', 'break', 'class', 

66 'continue', 'def', 'del', 'elif', 'else', 

67 'except', 'exec', 'finally', 'for', 'from', 

68 'global', 'if', 'import', 'in', 'is', 'lambda', 

69 'not', 'or', 'pass', 'print', 'raise', 'return', 

70 'try', 'while', 'with', 'yield', 'True', 'False', 

71 'None', 'eval', 'execfile', '__import__', 

72 '__package__') 

73 

74class Interpreter: 

75 """larch program compiler and interpreter. 

76 Thiso module compiles expressions and statements to AST representation, 

77 using python's ast module, and then executes the AST representation 

78 using a custom SymbolTable for named object (variable, functions). 

79 This then gives a restricted version of Python, with slightly modified 

80 namespace rules. The program syntax here is expected to be valid Python, 

81 but that may have been translated as with the inputText module. 

82 

83 The following Python syntax is not supported: 

84 Exec, Lambda, Class, Global, Generators, Yield, Decorators 

85 

86 In addition, Function is greatly altered so as to allow a Larch procedure. 

87 """ 

88 

89 supported_nodes = ('arg', 'assert', 'assign', 'attribute', 'augassign', 

90 'binop', 'boolop', 'break', 'bytes', 'call', 

91 'compare', 'constant', 'continue', 'delete', 'dict', 

92 'ellipsis', 'excepthandler', 'expr', 'expression', 

93 'extslice', 'for', 'functiondef', 'if', 'ifexp', 

94 'import', 'importfrom', 'index', 'interrupt', 

95 'list', 'listcomp', 'module', 'name', 

96 'nameconstant', 'num', 'pass', 'raise', 'repr', 

97 'return', 'slice', 'starred', 'str', 'subscript', 

98 'try', 'tryexcept', 'tryfinally', 'tuple', 

99 'unaryop', 'while', 

100 'set', 'setcomp', 'dictcomp', 

101 'with', 'formattedvalue', 'joinedstr') 

102 

103 def __init__(self, symtable=None, input=None, writer=None, 

104 historyfile=None, maxhistory=5000): 

105 self.symtable = symtable or SymbolTable(larch=self) 

106 

107 self.input = input or InputText(_larch=self, 

108 historyfile=historyfile, 

109 maxhistory=maxhistory) 

110 self.writer = writer or StdWriter(_larch=self) 

111 self._interrupt = None 

112 self.error = [] 

113 self.expr = None 

114 self.retval = None 

115 self._calldepth = 0 

116 self.func = None 

117 self.fname = '<stdin>' 

118 self.lineno = 0 

119 builtingroup = self.symtable._builtin 

120 mathgroup = self.symtable._math 

121 setattr(mathgroup, 'j', 1j) 

122 setattr(mathgroup, 'np', numpy) 

123 

124 # system-specific settings 

125 site_config.system_settings() 

126 from larch import builtins 

127 

128 for sym in builtins.from_math: 

129 setattr(mathgroup, sym, getattr(math, sym)) 

130 

131 for sym in builtins.from_builtin: 

132 setattr(builtingroup, sym, __builtins__[sym]) 

133 

134 for sym in builtins.from_numpy: 

135 val = getattr(numpy, sym, None) 

136 if val is not None: 

137 setattr(mathgroup, sym, val) 

138 

139 for fname, sym in builtins.numpy_renames.items(): 

140 val = getattr(numpy, sym, None) 

141 if val is not None: 

142 setattr(mathgroup, fname, val) 

143 

144 for name, value in builtins.constants.items(): 

145 setattr(mathgroup, name, value) 

146 

147 core_groups = ['_main', '_sys', '_builtin', '_math'] 

148 for groupname, entries in builtins.init_builtins.items(): 

149 if groupname not in core_groups: 

150 core_groups.append(groupname) 

151 if self.symtable.has_group(groupname): 

152 group = getattr(self.symtable, groupname, None) 

153 else: 

154 group = self.symtable.set_symbol(groupname, 

155 value=Group(__name__=groupname)) 

156 for fname, fcn in list(entries.items()): 

157 if callable(fcn): 

158 setattr(group, fname, 

159 Closure(func=fcn, _larch=self, _name=fname)) 

160 else: 

161 setattr(group, fname, fcn) 

162 

163 self.symtable._sys.core_groups = core_groups 

164 self.symtable._fix_searchGroups(force=True) 

165 

166 # set valid commands from builtins 

167 for cmd in builtins.valid_commands: 

168 self.symtable._sys.valid_commands.append(cmd) 

169 

170 # run any initialization routines 

171 for fcn in builtins.init_funcs: 

172 if callable(fcn): 

173 fcn(_larch=self) 

174 

175 for groupname, docstring in builtins.init_moddocs.items(): 

176 group = self.symtable.get_group(groupname) 

177 group.__doc__ = docstring 

178 

179 self.on_try = self.on_tryexcept 

180 self.on_tryfinally = self.on_tryexcept 

181 self.node_handlers = dict(((node, getattr(self, "on_%s" % node)) 

182 for node in self.supported_nodes)) 

183 

184 

185 def unimplemented(self, node): 

186 "unimplemented nodes" 

187 self.raise_exception(node, exc=NotImplementedError, 

188 msg="'%s' not supported" % (node.__class__.__name__)) 

189 

190 def raise_exception(self, node, exc=None, msg='', expr=None, 

191 fname=None, lineno=None, func=None): 

192 "add an exception" 

193 if self.error is None: 

194 self.error = [] 

195 if expr is None: 

196 expr = self.expr 

197 if fname is None: 

198 fname = self.fname 

199 if lineno is None: 

200 lineno = self.lineno 

201 

202 if func is None: 

203 func = self.func 

204 

205 if len(self.error) > 0 and not isinstance(node, ast.Module): 

206 msg = '%s' % msg 

207 err = LarchExceptionHolder(node=node, exc=exc, msg=msg, expr=expr, 

208 fname=fname, lineno=lineno, func=func) 

209 self._interrupt = ast.Raise() 

210 self.error.append(err) 

211 self.symtable._sys.last_error = err 

212 #raise RuntimeError 

213 

214 # main entry point for Ast node evaluation 

215 # parse: text of statements -> ast 

216 # run: ast -> result 

217 # eval: string statement -> result = run(parse(statement)) 

218 def parse(self, text, fname=None, lineno=-1): 

219 """parse statement/expression to Ast representation """ 

220 self.expr = text 

221 try: 

222 return ast.parse(text) 

223 except: 

224 etype, exc, tb = sys.exc_info() 

225 if (isinstance(exc, SyntaxError) and 

226 exc.msg == 'invalid syntax'): 

227 rwords = [] 

228 for word in PYTHON_RESERVED_WORDS: 

229 if (text.startswith('%s ' % word) or 

230 text.endswith(' %s' % word) or 

231 ' %s ' % word in text): 

232 rwords.append(word) 

233 if len(rwords) > 0: 

234 rwords = ", ".join(rwords) 

235 text = """May contain one of Python's reserved words: 

236 %s""" % (rwords) 

237 self.raise_exception(None, exc=SyntaxError, msg='Syntax Error', 

238 expr=text, fname=fname, lineno=lineno) 

239 

240 def run(self, node, expr=None, func=None, 

241 fname=None, lineno=None, with_raise=False): 

242 """executes parsed Ast representation for an expression""" 

243 # Note: keep the 'node is None' test: internal code here may run 

244 # run(None) and expect a None in return. 

245 out = None 

246 if len(self.error) > 0: 

247 return out 

248 if self.retval is not None: 

249 return self.retval 

250 if isinstance(self._interrupt, (ast.Break, ast.Continue)): 

251 return self._interrupt 

252 if node is None: 

253 return None 

254 if isinstance(node, str): 

255 node = self.parse(node) 

256 if lineno is not None: 

257 self.lineno = lineno 

258 if fname is not None: 

259 self.fname = fname 

260 if expr is not None: 

261 self.expr = expr 

262 # if func is not None: 

263 self.func = func 

264 

265 # get handler for this node: 

266 # on_xxx with handle nodes of type 'xxx', etc 

267 if node.__class__.__name__.lower() not in self.node_handlers: 

268 return self.unimplemented(node) 

269 handler = self.node_handlers[node.__class__.__name__.lower()] 

270 # run the handler: this will likely generate 

271 # recursive calls into this run method. 

272 try: 

273 out = handler(node) 

274 except: 

275 self.raise_exception(node, expr=self.expr, 

276 fname=self.fname, lineno=self.lineno) 

277 else: 

278 # enumeration objects are list-ified here... 

279 if isinstance(out, enumerate): 

280 out = list(out) 

281 return out 

282 

283 def __call__(self, expr, **kw): 

284 return self.eval(expr, **kw) 

285 

286 def eval(self, expr, fname=None, lineno=0, add_history=True): 

287 """evaluates an expression 

288 really: puts expression to input buffer, and if the 

289 input buffer is complete, it executes all the code in 

290 that buffer. 

291 """ 

292 self.input.put(expr, filename=fname, lineno=lineno, 

293 add_history=add_history) 

294 if self.input.complete and len(self.input) > 0: 

295 return self.execute_input() 

296 

297 def execute_input(self): 

298 """executes the text in the input buffer""" 

299 self.error = [] 

300 if not hasattr(self.symtable._sys, 'call_stack'): 

301 self.symtable._sys.call_stack = [] 

302 call_stack = self.symtable._sys.call_stack 

303 call_stack.append(None) 

304 

305 ret = None 

306 

307 while len(self.input) > 0: 

308 text, fname, lineno = self.input.get() 

309 # print("EXEC ", text, fname, lineno) 

310 # self.input.buffer.append(text) 

311 if len(self.input.curtext) > 0 or len(self.input.blocks) > 0: 

312 continue 

313 call_stack[-1] = (text, fname, lineno) 

314 try: 

315 node = self.parse(text, fname=fname, lineno=lineno) 

316 ret = self.run(node, expr=text, fname=fname, lineno=lineno) 

317 except RuntimeError: 

318 pass 

319 

320 self.input.clear() 

321 call_stack.pop() 

322 return ret 

323 

324 def runfile(self, filename, new_module=None): 

325 """ 

326 run the larch code held in a file, possibly as 'module' 

327 """ 

328 ret = self.input.putfile(filename) 

329 if ret is not None: 

330 exc, msg = ret 

331 err = LarchExceptionHolder(node=None, exc=IOError, 

332 msg='Cannot read %s' % filename) 

333 self.error.append(err) 

334 self.symtable._sys.last_error = err 

335 return 

336 

337 # self.input.put(text, filename=filename, lineno=0, add_history=add_history) 

338 if not self.input.complete: 

339 msg = "File '%s' ends with incomplete input" % (filename) 

340 text = None 

341 lineno = 0 

342 if len(self.input.blocks) > 0 and filename is not None: 

343 blocktype, lineno, text = self.input.blocks[0] 

344 msg = "File '%s' ends with un-terminated '%s'" % (filename, 

345 blocktype) 

346 elif self.input.saved_text is not BLANK_TEXT: 

347 text, fname, lineno = self.input.saved_text 

348 msg = "File '%s' ends with incomplete statement" % (filename) 

349 self.input.clear() 

350 err = LarchExceptionHolder(node=None, exc=SyntaxError, msg=msg, 

351 expr=text, fname=filename, 

352 lineno=lineno) 

353 

354 self.error.append(err) 

355 self.symtable._sys.last_error = err 

356 return 

357 

358 thismod = None 

359 if new_module is not None: 

360 # save current module group 

361 # create new group, set as moduleGroup and localGroup 

362 self.symtable.save_frame() 

363 thismod = self.symtable.create_group(name=new_module) 

364 self.symtable._sys.modules[new_module] = thismod 

365 self.symtable.set_frame((thismod, thismod)) 

366 

367 ret = self.execute_input() 

368 

369 if new_module is not None: 

370 # for a "newly created module" (as on import), 

371 # the module group is the return value 

372 self.symtable.restore_frame() 

373 return thismod 

374 

375 

376 

377 def show_errors(self): 

378 """show errors """ 

379 if self.error: 

380 call_stack = self.symtable._sys.call_stack 

381 writer = self.writer 

382 fname = self.fname 

383 lineno = self.lineno 

384 

385 writer.write('Traceback (most recent calls last): \n') 

386 for eblock, efname, elineno in call_stack: 

387 text = "File %s, line %i" % (efname, elineno) 

388 if efname != fname and elineno != lineno: 

389 text = "%s\n %s" % (text, eblock.split('\n')[0]) 

390 writer.write(' %s\n' % (text)) 

391 

392 errors_seen = [] 

393 for err in self.error: 

394 exc_name, errmsg = err.get_error() 

395 file_lineno = errmsg.split('\n')[0].strip() 

396 if file_lineno in errors_seen: 

397 continue 

398 errors_seen.append(file_lineno) 

399 writer.write(errmsg) 

400 self.error = [] 

401 

402 def run_init_scripts(self): 

403 for fname in site_config.init_files: 

404 if Path(fname).exists(): 

405 try: 

406 self.runfile(fname) 

407 except: 

408 self.raise_exception(None, exc=RuntimeError, 

409 msg='Initialization Error') 

410 

411 def dump(self, node, **kw): 

412 "simple ast dumper" 

413 return ast.dump(node, **kw) 

414 

415 # handlers for ast components 

416 def on_expr(self, node): 

417 "expression" 

418 return self.run(node.value) # ('value',) 

419 

420 def on_index(self, node): 

421 "index" 

422 return self.run(node.value) # ('value',) 

423 

424 def on_return(self, node): # ('value',) 

425 "return statement: look for None, return special sentinal" 

426 if self._calldepth == 0: 

427 raise SyntaxError('cannot return at top level') 

428 ret = self.run(node.value) 

429 self.retval = ret if ret is not None else ReturnedNone 

430 return 

431 

432 def on_repr(self, node): 

433 "repr " 

434 return repr(self.run(node.value)) # ('value',) 

435 

436 def on_module(self, node): # ():('body',) 

437 "module def" 

438 out = None 

439 for tnode in node.body: 

440 out = self.run(tnode) 

441 return out 

442 

443 def on_expression(self, node): 

444 "basic expression" 

445 return self.on_module(node) # ():('body',) 

446 

447 def on_pass(self, node): 

448 "pass statement" 

449 return None # () 

450 

451 def on_ellipsis(self, node): 

452 "ellipses" 

453 return Ellipsis 

454 

455 # for break and continue: set the instance variable _interrupt 

456 def on_interrupt(self, node): # () 

457 "interrupt handler" 

458 self._interrupt = node 

459 return node 

460 

461 def on_break(self, node): 

462 "break" 

463 return self.on_interrupt(node) 

464 

465 def on_continue(self, node): 

466 "continue" 

467 return self.on_interrupt(node) 

468 

469 def on_arg(self, node): 

470 "arg for function definitions" 

471 return node.arg 

472 

473 def on_assert(self, node): # ('test', 'msg') 

474 "assert statement" 

475 testval = self.run(node.test) 

476 if not testval: 

477 msg = node.msg.s if node.msg else "" 

478 self.raise_exception(node, exc=AssertionError, msg=msg) 

479 return True 

480 

481 def on_list(self, node): # ('elt', 'ctx') 

482 "list" 

483 return [self.run(e) for e in node.elts] 

484 

485 def on_tuple(self, node): # ('elts', 'ctx') 

486 "tuple" 

487 return tuple(self.on_list(node)) 

488 

489 def on_set(self, node): # ('elts') 

490 """Set.""" 

491 return set([self.run(k) for k in node.elts]) 

492 

493 def on_dict(self, node): # ('keys', 'values') 

494 "dictionary" 

495 nodevals = list(zip(node.keys, node.values)) 

496 run = self.run 

497 return dict([(run(k), run(v)) for k, v in nodevals]) 

498 

499 def on_constant(self, node): # ('value', 'kind') 

500 """Return constant value.""" 

501 return node.value 

502 

503 def on_num(self, node): 

504 'return number' 

505 return node.n # ('n',) 

506 

507 def on_str(self, node): 

508 'return string' 

509 return node.s # ('s',) 

510 

511 def on_bytes(self, node): 

512 'return bytes' 

513 return node.s # ('s',) 

514 

515 def on_joinedstr(self, node): # ('values',) 

516 "join strings, used in f-strings" 

517 return ''.join([self.run(k) for k in node.values]) 

518 

519 def on_formattedvalue(self, node): # ('value', 'conversion', 'format_spec') 

520 "formatting used in f-strings" 

521 val = self.run(node.value) 

522 fstring_converters = {115: str, 114: repr, 97: ascii} 

523 if node.conversion in fstring_converters: 

524 val = fstring_converters[node.conversion](val) 

525 fmt = '{0}' 

526 if node.format_spec is not None: 

527 fmt = f'{ 0:{self.run(node.format_spec)}} ' 

528 return fmt.format(val) 

529 

530 def on_nameconstant(self, node): # ('value') 

531 """ Name Constant node (new in Python3.4)""" 

532 return node.value 

533 

534 def on_name(self, node): # ('id', 'ctx') 

535 """ Name node """ 

536 ctx = node.ctx.__class__ 

537 if ctx == ast.Del: 

538 val = self.symtable.del_symbol(node.id) 

539 elif ctx == ast.Param: # for Function Def 

540 val = str(node.id) 

541 else: 

542 try: 

543 val = self.symtable.get_symbol(node.id) 

544 except (NameError, LookupError): 

545 msg = "name '%s' is not defined" % node.id 

546 val = None 

547 self.raise_exception(node, msg=msg) 

548 return val 

549 

550 def on_starred(self, node): # ('value', 'ctx') 

551 """ Starred node """ 

552 ctx = node.ctx.__class__ 

553 if ctx != ast.Load: 

554 msg = "can only load starargs" 

555 self.raise_exception(node, msg=msg) 

556 return self.run(node.value) 

557 

558 def node_assign(self, node, val): 

559 """here we assign a value (not the node.value object) to a node 

560 this is used by on_assign, but also by for, list comprehension, etc. 

561 """ 

562 if len(self.error) > 0: 

563 return 

564 if node.__class__ == ast.Name: 

565 if not valid_symbol_name(node.id): 

566 errmsg = f"invalid symbol name (reserved word?) {node.id}" 

567 self.raise_exception(node, exc=NameError, msg=errmsg) 

568 sym = self.symtable.set_symbol(node.id, value=val) 

569 elif node.__class__ == ast.Attribute: 

570 if node.ctx.__class__ == ast.Load: 

571 errmsg = "cannot assign to attribute %s" % node.attr 

572 self.raise_exception(node, exc=AttributeError, msg=errmsg) 

573 

574 setattr(self.run(node.value), node.attr, val) 

575 

576 elif node.__class__ == ast.Subscript: 

577 sym = self.run(node.value) 

578 xslice = self.run(node.slice) 

579 if isinstance(node.slice, ast.Slice): 

580 i = xslice.start 

581 sym[slice(xslice.start, xslice.stop)] = val 

582 else: 

583 sym[xslice] = val 

584 elif node.__class__ in (ast.Tuple, ast.List): 

585 if len(val) == len(node.elts): 

586 for telem, tval in zip(node.elts, val): 

587 self.node_assign(telem, tval) 

588 else: 

589 raise ValueError('too many values to unpack') 

590 

591 def on_attribute(self, node): # ('value', 'attr', 'ctx') 

592 "extract attribute" 

593 ctx = node.ctx.__class__ 

594 if ctx == ast.Del: 

595 return delattr(sym, node.attr) 

596 

597 sym = self.run(node.value) 

598 if node.attr not in UNSAFE_ATTRS: 

599 try: 

600 return getattr(sym, node.attr) 

601 except AttributeError: 

602 pass 

603 

604 obj = self.run(node.value) 

605 fmt = "%s does not have member '%s'" 

606 if not isgroup(obj): 

607 obj = obj.__class__ 

608 fmt = "%s does not have attribute '%s'" 

609 msg = fmt % (obj, node.attr) 

610 self.raise_exception(node, exc=AttributeError, msg=msg) 

611 

612 def on_assign(self, node): # ('targets', 'value') 

613 "simple assignment" 

614 val = self.run(node.value) 

615 if len(self.error) > 0: 

616 return 

617 for tnode in node.targets: 

618 self.node_assign(tnode, val) 

619 return 

620 

621 def on_augassign(self, node): # ('target', 'op', 'value') 

622 "augmented assign" 

623 val = ast.BinOp(left=node.target, op=node.op, right=node.value) 

624 return self.on_assign(ast.Assign(targets=[node.target], value=val)) 

625 

626 def on_slice(self, node): # ():('lower', 'upper', 'step') 

627 "simple slice" 

628 return slice(self.run(node.lower), self.run(node.upper), 

629 self.run(node.step)) 

630 

631 def on_extslice(self, node): # ():('dims',) 

632 "extended slice" 

633 return tuple([self.run(tnode) for tnode in node.dims]) 

634 

635 def on_subscript(self, node): # ('value', 'slice', 'ctx') 

636 "subscript handling -- one of the tricky parts" 

637 val = self.run(node.value) 

638 nslice = self.run(node.slice) 

639 ctx = node.ctx.__class__ 

640 if ctx in (ast.Load, ast.Store): 

641 return val[nslice] 

642 else: 

643 msg = "subscript with unknown context" 

644 self.raise_exception(node, msg=msg) 

645 

646 def on_delete(self, node): # ('targets',) 

647 "delete statement" 

648 for tnode in node.targets: 

649 if tnode.ctx.__class__ != ast.Del: 

650 break 

651 if tnode.__class__ == ast.Subscript: 

652 try: 

653 parent = self.run(tnode.value) 

654 if hasattr(parent, '__delitem__') and tnode.slice.__class__ == ast.Constant: 

655 child = tnode.slice.value 

656 parent.__delitem__(child) 

657 except: 

658 msg = "could not delete symbol" 

659 self.raise_exception(node, msg=msg) 

660 else: 

661 children = [] 

662 while tnode.__class__ == ast.Attribute: 

663 children.append(tnode.attr) 

664 tnode = tnode.value 

665 if tnode.__class__ == ast.Name: 

666 children.append(tnode.id) 

667 children.reverse() 

668 self.symtable.del_symbol('.'.join(children)) 

669 else: 

670 msg = "could not delete symbol" 

671 self.raise_exception(node, msg=msg) 

672 

673 def on_unaryop(self, node): # ('op', 'operand') 

674 "unary operator" 

675 return OPERATORS[node.op.__class__](self.run(node.operand)) 

676 

677 def on_binop(self, node): # ('left', 'op', 'right') 

678 "binary operator" 

679 return OPERATORS[node.op.__class__](self.run(node.left), 

680 self.run(node.right)) 

681 

682 def on_boolop(self, node): # ('op', 'values') 

683 "boolean operator" 

684 val = self.run(node.values[0]) 

685 is_and = ast.And == node.op.__class__ 

686 if (is_and and val) or (not is_and and not val): 

687 for n in node.values[1:]: 

688 val = OPERATORS[node.op.__class__](val, self.run(n)) 

689 if (is_and and not val) or (not is_and and val): 

690 break 

691 return val 

692 

693 def on_compare(self, node): # ('left', 'ops', 'comparators') 

694 "comparison operators" 

695 lval = self.run(node.left) 

696 out = True 

697 for oper, rnode in zip(node.ops, node.comparators): 

698 comp = OPERATORS[oper.__class__] 

699 rval = self.run(rnode) 

700 out = comp(lval, rval) 

701 lval = rval 

702 if not hasattr(out, 'any') and not out: 

703 break 

704 return out 

705 

706 def on_printOLD(self, node): # ('dest', 'values', 'nl') 

707 """ note: implements Python2 style print statement, not 

708 print() function. Probably, the 'larch2py' translation 

709 should look for and translate print -> print_() to become 

710 a customized function call. 

711 """ 

712 dest = self.run(node.dest) or self.writer 

713 end = '' 

714 if node.nl: 

715 end = '\n' 

716 out = [self.run(tnode) for tnode in node.values] 

717 if out and len(self.error)==0: 

718 print(*out, file=dest, end=end) 

719 

720 def on_if(self, node): # ('test', 'body', 'orelse') 

721 "regular if-then-else statement" 

722 block = node.body 

723 if not self.run(node.test): 

724 block = node.orelse 

725 for tnode in block: 

726 self.run(tnode) 

727 

728 def on_ifexp(self, node): # ('test', 'body', 'orelse') 

729 "if expressions" 

730 expr = node.orelse 

731 if self.run(node.test): 

732 expr = node.body 

733 return self.run(expr) 

734 

735 def on_with(self, node): # ('items', 'body', 'type_comment') 

736 """with blocks.""" 

737 contexts = [] 

738 for item in node.items: 

739 ctx = self.run(item.context_expr) 

740 contexts.append(ctx) 

741 if hasattr(ctx, '__enter__'): 

742 result = ctx.__enter__() 

743 if item.optional_vars is not None: 

744 self.node_assign(item.optional_vars, result) 

745 else: 

746 msg = "object does not support the context manager protocol" 

747 raise TypeError(f"'{type(ctx)}' {msg}") 

748 for bnode in node.body: 

749 self.run(bnode) 

750 if self._interrupt is not None: 

751 break 

752 

753 for ctx in contexts: 

754 if hasattr(ctx, '__exit__'): 

755 ctx.__exit__() 

756 

757 

758 def on_while(self, node): # ('test', 'body', 'orelse') 

759 "while blocks" 

760 while self.run(node.test): 

761 self._interrupt = None 

762 for tnode in node.body: 

763 self.run(tnode) 

764 if self._interrupt is not None: 

765 break 

766 if isinstance(self._interrupt, ast.Break): 

767 break 

768 else: 

769 for tnode in node.orelse: 

770 self.run(tnode) 

771 self._interrupt = None 

772 

773 def on_for(self, node): # ('target', 'iter', 'body', 'orelse') 

774 "for blocks" 

775 for val in self.run(node.iter): 

776 self.node_assign(node.target, val) 

777 if len(self.error) > 0: 

778 return 

779 self._interrupt = None 

780 for tnode in node.body: 

781 self.run(tnode) 

782 if len(self.error) > 0: 

783 return 

784 if self._interrupt is not None: 

785 break 

786 if isinstance(self._interrupt, ast.Break): 

787 break 

788 else: 

789 for tnode in node.orelse: 

790 self.run(tnode) 

791 self._interrupt = None 

792 

793 

794 def comprehension_data(self, node): # ('elt', 'generators') 

795 "data for comprehensions" 

796 mylocals = {} 

797 saved_syms = {} 

798 

799 for tnode in node.generators: 

800 if tnode.__class__ == ast.comprehension: 

801 if tnode.target.__class__ == ast.Name: 

802 if not valid_symbol_name(tnode.target.id): 

803 errmsg = f"invalid symbol name (reserved word?) {tnode.target.id}" 

804 self.raise_exception(tnode.target, exc=NameError, msg=errmsg) 

805 mylocals[tnode.target.id] = [] 

806 if self.symtable.has_symbol(tnode.target.id): 

807 saved_syms[tnode.target.id] = deepcopy(self.symtable.get_symbol(tnode.target.id)) 

808 

809 elif tnode.target.__class__ == ast.Tuple: 

810 target = [] 

811 for tval in tnode.target.elts: 

812 mylocals[tval.id] = [] 

813 if self.symtable.has_symbol(tval.id): 

814 saved_syms[tval.id] = deepcopy(self.symtable.get_symbol(tval.id)) 

815 

816 for tnode in node.generators: 

817 if tnode.__class__ == ast.comprehension: 

818 ttype = 'name' 

819 if tnode.target.__class__ == ast.Name: 

820 if not valid_symbol_name(tnode.target.id): 

821 errmsg = f"invalid symbol name (reserved word?) {tnode.target.id}" 

822 self.raise_exception(tnode.target, exc=NameError, msg=errmsg) 

823 ttype, target = 'name', tnode.target.id 

824 elif tnode.target.__class__ == ast.Tuple: 

825 ttype = 'tuple' 

826 target =tuple([tval.id for tval in tnode.target.elts]) 

827 

828 for val in self.run(tnode.iter): 

829 if ttype == 'name': 

830 self.symtable.set_symbol(target, val) 

831 else: 

832 for telem, tval in zip(target, val): 

833 self.symtable.set_symbol(target, val) 

834 

835 add = True 

836 for cond in tnode.ifs: 

837 add = add and self.run(cond) 

838 if add: 

839 if ttype == 'name': 

840 mylocals[target].append(val) 

841 else: 

842 for telem, tval in zip(target, val): 

843 mylocals[telem].append(tval) 

844 return mylocals, saved_syms 

845 

846 def on_listcomp(self, node): 

847 """List comprehension""" 

848 mylocals, saved_syms = self.comprehension_data(node) 

849 

850 names = list(mylocals.keys()) 

851 data = list(mylocals.values()) 

852 def listcomp_recurse(out, i, names, data): 

853 if i == len(names): 

854 out.append(self.run(node.elt)) 

855 return 

856 

857 for val in data[i]: 

858 self.symtable.set_symbol(names[i], val) 

859 listcomp_recurse(out, i+1, names, data) 

860 

861 out = [] 

862 listcomp_recurse(out, 0, names, data) 

863 for name, val in saved_syms.items(): 

864 self.symtable.set_symbol(name, val) 

865 return out 

866 

867 def on_setcomp(self, node): 

868 """Set comprehension""" 

869 return set(self.on_listcomp(node)) 

870 

871 def on_dictcomp(self, node): 

872 """Dictionary comprehension""" 

873 mylocals, saved_syms = self.comprehension_data(node) 

874 

875 names = list(mylocals.keys()) 

876 data = list(mylocals.values()) 

877 

878 def dictcomp_recurse(out, i, names, data): 

879 if i == len(names): 

880 out[self.run(node.key)] = self.run(node.value) 

881 return 

882 

883 for val in data[i]: 

884 self.symtable.set_symbol(names[i], val) 

885 dictcomp_recurse(out, i+1, names, data) 

886 

887 out = {} 

888 dictcomp_recurse(out, 0, names, data) 

889 for name, val in saved_syms.items(): 

890 self.symtable.set_symbol(name, val) 

891 return out 

892 

893 def on_excepthandler(self, node): # ('type', 'name', 'body') 

894 "exception handler..." 

895 # print("except handler %s / %s " % (node.type, ast.dump(node.name))) 

896 return (self.run(node.type), node.name, node.body) 

897 

898 def on_tryexcept(self, node): # ('body', 'handlers', 'orelse') 

899 "try/except blocks" 

900 no_errors = True 

901 for tnode in node.body: 

902 self.run(tnode) 

903 no_errors = no_errors and len(self.error) == 0 

904 if self.error: 

905 e_type, e_value, e_tb = self.error[-1].exc_info 

906 this_exc = e_type() 

907 

908 for hnd in node.handlers: 

909 htype = None 

910 if hnd.type is not None: 

911 htype = __builtins__.get(hnd.type.id, None) 

912 if htype is None or isinstance(this_exc, htype): 

913 self.error = [] 

914 self._interrupt = None 

915 if hnd.name is not None: 

916 self.node_assign(hnd.name, e_value) 

917 for tline in hnd.body: 

918 self.run(tline) 

919 break 

920 if no_errors and hasattr(node, 'orelse'): 

921 for tnode in node.orelse: 

922 self.run(tnode) 

923 

924 if hasattr(node, 'finalbody'): 

925 for tnode in node.finalbody: 

926 self.run(tnode) 

927 

928 def on_raise(self, node): # ('type', 'inst', 'tback') 

929 "raise statement" 

930 out = self.run(node.exc) 

931 msg = ' '.join(out.args) 

932 msg2 = self.run(node.cause) 

933 if msg2 not in (None, 'None'): 

934 msg = "%s: %s" % (msg, msg2) 

935 self.raise_exception(None, exc=out.__class__, msg=msg, expr='') 

936 

937 def on_call(self, node): 

938 "function/procedure execution" 

939 # ('func', 'args', 'keywords', 'starargs', 'kwargs') 

940 func = self.run(node.func) 

941 if not callable(func): 

942 msg = "'%s' is not callable!!" % (func) 

943 self.raise_exception(node, exc=TypeError, msg=msg) 

944 

945 args = [self.run(targ) for targ in node.args] 

946 starargs = getattr(node, 'starargs', None) 

947 if starargs is not None: 

948 args = args + self.run(starargs) 

949 

950 keywords = {} 

951 if func == print: 

952 keywords['file'] = self.writer 

953 for key in node.keywords: 

954 if not isinstance(key, ast.keyword): 

955 msg = "keyword error in function call '%s'" % (func) 

956 self.raise_exception(node, msg=msg) 

957 if key.arg is None: 

958 keywords.update(self.run(key.value)) 

959 elif key.arg in keywords: 

960 self.raise_exception(node, 

961 msg="keyword argument repeated: %s" % key.arg, 

962 exc=SyntaxError) 

963 else: 

964 keywords[key.arg] = self.run(key.value) 

965 

966 kwargs = getattr(node, 'kwargs', None) 

967 if kwargs is not None: 

968 keywords.update(self.run(kwargs)) 

969 

970 if isinstance(func, Procedure): 

971 self._calldepth += 1 

972 try: 

973 out = func(*args, **keywords) 

974 except Exception as ex: 

975 out = None 

976 func_name = getattr(func, '__name__', str(func)) 

977 self.raise_exception( 

978 node, msg="Error running function call '%s' with args %s and " 

979 "kwargs %s: %s" % (func_name, args, keywords, ex)) 

980 finally: 

981 if isinstance(func, Procedure): 

982 self._calldepth -= 1 

983 return out 

984 

985 def on_functiondef(self, node): 

986 "define procedures" 

987 # ('name', 'args', 'body', 'decorator_list') 

988 if node.decorator_list != []: 

989 raise Warning("decorated procedures not supported!") 

990 kwargs = [] 

991 offset = len(node.args.args) - len(node.args.defaults) 

992 for idef, defnode in enumerate(node.args.defaults): 

993 defval = self.run(defnode) 

994 keyval = self.run(node.args.args[idef+offset]) 

995 kwargs.append((keyval, defval)) 

996 # kwargs.reverse() 

997 args = [tnode.arg for tnode in node.args.args[:offset]] 

998 doc = None 

999 if (isinstance(node.body[0], ast.Expr) and 

1000 isinstance(node.body[0].value, ast.Constant)): 

1001 docnode = node.body[0] 

1002 doc = docnode.value.value 

1003 

1004 vararg = self.run(node.args.vararg) 

1005 varkws = self.run(node.args.kwarg) 

1006 proc = Procedure(node.name, _larch=self, doc= doc, 

1007 body = node.body, 

1008 fname = self.fname, 

1009 lineno = self.lineno, 

1010 args = args, 

1011 kwargs = kwargs, 

1012 vararg = vararg, 

1013 varkws = varkws) 

1014 self.symtable.set_symbol(node.name, value=proc) 

1015 

1016 # imports 

1017 def on_import(self, node): # ('names',) 

1018 "simple import" 

1019 for tnode in node.names: 

1020 self.import_module(tnode.name, asname=tnode.asname) 

1021 

1022 def on_importfrom(self, node): # ('module', 'names', 'level') 

1023 "import/from" 

1024 fromlist, asname = [], [] 

1025 for tnode in node.names: 

1026 fromlist.append(tnode.name) 

1027 asname.append(tnode.asname) 

1028 self.import_module(node.module, 

1029 asname=asname, fromlist=fromlist) 

1030 

1031 

1032 def import_module(self, name, asname=None, 

1033 fromlist=None, do_reload=False): 

1034 """ 

1035 import a module (larch or python), installing it into the symbol table. 

1036 required arg: 

1037 name name of module to import 

1038 'foo' in 'import foo' 

1039 options: 

1040 fromlist list of symbols to import with 'from-import' 

1041 ['x','y'] in 'from foo import x, y' 

1042 asname alias for imported name(s) 

1043 'bar' in 'import foo as bar' 

1044 or 

1045 ['s','t'] in 'from foo import x as s, y as t' 

1046 

1047 this method covers a lot of cases (larch or python, import 

1048 or from-import, use of asname) and so is fairly long. 

1049 """ 

1050 st_sys = self.symtable._sys 

1051 for idir in st_sys.path: 

1052 if idir not in sys.path and Path(idir).exists(): 

1053 sys.path.append(idir) 

1054 

1055 # step 1 import the module to a global location 

1056 # either sys.modules for python modules 

1057 # or st_sys.modules for larch modules 

1058 # reload takes effect here in the normal python way: 

1059 

1060 thismod = None 

1061 if name in st_sys.modules: 

1062 thismod = st_sys.modules[name] 

1063 elif name in sys.modules: 

1064 thismod = sys.modules[name] 

1065 if thismod is None or do_reload: 

1066 # first look for "name.lar" 

1067 islarch = False 

1068 larchname = "%s.lar" % name 

1069 for dirname in st_sys.path: 

1070 if not Path(dirname).exists(): 

1071 continue 

1072 if larchname in sorted(os.listdir(dirname)): 

1073 islarch = True 

1074 modname = Path(dirname, larchname).absolute().as_posix() 

1075 try: 

1076 thismod = self.runfile(modname, new_module=name) 

1077 except: 

1078 thismod = None 

1079 

1080 # we found a module with the right name, 

1081 # so break out of loop, even if there was an error. 

1082 break 

1083 

1084 if len(self.error) > 0 and name in st_sys.modules: 

1085 st_sys.modules.pop(name) 

1086 

1087 # or, if not a larch module, load as a regular python module 

1088 if thismod is None and not islarch and name not in sys.modules: 

1089 try: 

1090 __import__(name) 

1091 thismod = sys.modules[name] 

1092 except: 

1093 thismod = None 

1094 

1095 if thismod is None: 

1096 self.raise_exception(None, exc=ImportError, msg='Import Error') 

1097 return 

1098 

1099 # now we install thismodule into the current moduleGroup 

1100 # import full module 

1101 if fromlist is None: 

1102 if asname is None: 

1103 asname = name 

1104 parts = asname.split('.') 

1105 asname = parts.pop() 

1106 targetgroup = st_sys.moduleGroup 

1107 while len(parts) > 0: 

1108 subname = parts.pop(0) 

1109 subgrp = Group() 

1110 setattr(targetgroup, subname, subgrp) 

1111 targetgroup = subgrp 

1112 setattr(targetgroup, asname, thismod) 

1113 # import-from construct 

1114 else: 

1115 if asname is None: 

1116 asname = [None]*len(fromlist) 

1117 targetgroup = st_sys.moduleGroup 

1118 for sym, alias in zip(fromlist, asname): 

1119 if alias is None: 

1120 alias = sym 

1121 setattr(targetgroup, alias, getattr(thismod, sym)) 

1122 # end of import_module