topo.command.basic

1 """ 2 High-level user-level commands controlling the entire simulation. 3 $Id: basic.py 11324 2010-07-28 17:12:56Z ceball $ 4 """ 5 __version__='$Revision: 11324 $' 6 7 import cPickle as pickle 8 9 from xml.parsers.expat import ExpatError 10 11 import os,sys,re,string,time,platform 12 13 import __main__ 14 15 # gzip module might not have been built (if zlib could not be found when building) 16 try: 17 import gzip 18 except ImportError: 19 pass 20 21 import param 22 from param.parameterized import PicklableClassAttributes, ParameterizedFunction 23 from param.parameterized import ParamOverrides 24 from param.external import OrderedDict 25 from param import normalize_path 26 27 import topo 28 from topo.base.sheet import Sheet 29 from topo.base.projection import ProjectionSheet 30 from topo.sheet import GeneratorSheet 31 from topo.misc.util import MultiFile 32 from topo.misc.picklemain import PickleMain 33 from topo.misc.genexamples import generate as _generate 34 from topo.base.functionfamily import PatternDrivenAnalysis

35 36 37 -def generate_example(target):

38 """ 39 Generate the saved network target, as defined in 40 topo.misc.genexamples. 41 """ 42 _generate(targets=[target])

43 44 45 # Not sure where to put CommandMetaclass, since it doesn't need to be 46 # seen or used by anyone. Probably also the import error classes 47 # shouldn't be so visible. 48 from param.parameterized import ParameterizedMetaclass

49 -class CommandMetaclass(ParameterizedMetaclass):

50 """ 51 A class having this as a metaclass will have its __call__() method 52 automatically wrapped so that any exception occurring inside 53 __call__() will be passed to the class's _except() method. 54 """

55 - def __new__(mcs,classname,bases,classdict):

56 if '__call__' in classdict: 57 classdict['__call__'] = mcs._safecall(classdict['__call__']) 58 #assert '_except' in classdic 59 # else it is probably abstract, or something 60 return ParameterizedMetaclass.__new__(mcs,classname,bases,classdict)

61 62 @classmethod

63 - def _safecall(mcs,fn):

64 """ 65 Wrap fn with caller, which catches any exception raised inside 66 fn() and passes it to _except(). 67 """ 68 def caller(self,*args,**kw): 69 try: 70 return fn(self,*args,**kw) 71 except Exception, e: 72 # Mis-invoked call should raise error as normal. 73 if isinstance(e,TypeError): 74 import inspect 75 # Is this a hack to detect mis-calling? 76 if len(inspect.getargspec(fn)[0])!=len(args): 77 raise 78 try: 79 return self._except(e) 80 except: 81 # the _except() method raises an error (or doesn't 82 # exist). what should happen? programming error, 83 # so I guess just re-raise 84 raise

85 return caller 86

87 88 89 -class Command(ParameterizedFunction):

90 """ 91 Parameterized command: any error when the command is run (called) 92 will not raise an exception, but will instead generate a warning. 93 """ 94 __metaclass__ = CommandMetaclass 95 __abstract = True 96

97 - def _except(self,e):

98 # import traceback 99 # print traceback.print_exc() 100 self.warning("%s failed: %s"%(self,e)) # or traceback.format_exc()))

101

102 - def __call__(self,*args,**params):

103 return super(Command,self).__call__(*args,**params)

104

105 106 -class ImportErrorObject(object):

107 """ 108 Raises an ImportError on any attempt to access an attribute, call, 109 or get an item. 110 111 Useful to delay an ImportError until the point of use, thus 112 allowing e.g. a class attribute to contain something from a 113 non-core external module (e.g. pylab). 114 115 Delaying an ImportError until the point of use allows users to be 116 informed of the possibility of having various extra functions on 117 installation of a missing package. 118 """

119 - def __init__(self,module_name):

120 self.__module_name = module_name

121 - def _raise(self):

122 #param.Parameterized().warning("err:%s"%self.module_name) 123 raise ImportError, "No module named %s. Install %s to get this functionality."%(self.__module_name,self.__module_name) 124 return None

125 - def __call__(self,*args,**kw):

126 self._raise()

127 # Might be better to override __getattribute__, special casing the 128 # module_name attribute. Then everything is guaranteed to raise an 129 # error (rather than covering call, getitem, getattr, and maybe 130 # other things I've forgotten about).

131 - def __getattr__(self,name):

132 return self._raise()

133 - def __getitem__(self,i):

134 self._raise()

135

136 137 138 -class ImportErrorRaisingFakeModule(object):

139 """ 140 Returns an ImportErrorObject for any attribute request. 141 142 Instances of this class can be used in place of a module to delay 143 an import error until the point of use of an attribute of that 144 module. 145 146 See ImportErrorObject for more details. 147 """

148 - def __init__(self,module_name):

149 self.__module_name = module_name

150 - def __getattr__(self,name):

151 return ImportErrorObject(self.__module_name)

152

153 154 # CEBALERT: commands in here should inherit from Command, and make use 155 # of _except() to ensure all necessary state is reverted. 156 157 -def save_input_generators():

158 """Save a copy of the active_sim's current input_generators for all GeneratorSheets.""" 159 # ensure EPs get started (if save_input_generators is called before the simulation is run()) 160 topo.sim.run(0.0) 161 162 generator_sheets = topo.sim.objects(GeneratorSheet).values() 163 for sheet in generator_sheets: 164 sheet.push_input_generator()

165

166 167 -def restore_input_generators():

168 """Restore previously saved input_generators for all of topo.sim's GeneratorSheets.""" 169 generator_sheets = topo.sim.objects(GeneratorSheet).values() 170 for sheet in generator_sheets: 171 sheet.pop_input_generator()

172

173 174 -def clear_event_queue():

175 """Remove pending events from the simulator's event queue.""" 176 topo.sim.event_clear()

177

178 179 -def pattern_present(inputs={},duration=1.0,plastic=False,overwrite_previous=False,apply_output_fns=True):

180 """ 181 Present the specified test patterns for the specified duration. 182 183 Given a set of input patterns (dictionary of 184 GeneratorSheetName:PatternGenerator pairs), installs them into the 185 specified GeneratorSheets, runs the simulation for the specified 186 length of time, then restores the original patterns and the 187 original simulation time. Thus this input is not considered part 188 of the regular simulation, and is usually for testing purposes. 189 190 As a special case, if 'inputs' is just a single pattern, and not 191 a dictionary, it is presented to all GeneratorSheets. 192 193 If a simulation is not provided, the active simulation, if one 194 exists, is requested. 195 196 If this process is interrupted by the user, the temporary patterns 197 may still be installed on the retina. 198 199 If overwrite_previous is true, the given inputs overwrite those 200 previously defined. 201 202 If plastic is False, overwrites the existing values of Sheet.plastic 203 to disable plasticity, then reenables plasticity. 204 205 In order to to see the sequence of values presented, use the back arrow 206 history mechanism in the GUI. Note that the GUI's Activity window must 207 be open and the display parameter set to true (display=True). 208 """ 209 # ensure EPs get started (if pattern_present is called before the simulation is run()) 210 topo.sim.run(0.0) 211 212 213 if not overwrite_previous: 214 save_input_generators() 215 216 if not plastic: 217 # turn off plasticity everywhere 218 for sheet in topo.sim.objects(Sheet).values(): 219 sheet.override_plasticity_state(new_plasticity_state=False) 220 221 if not apply_output_fns: 222 for each in topo.sim.objects(Sheet).values(): 223 if hasattr(each,'measure_maps'): 224 if each.measure_maps: 225 each.apply_output_fns = False 226 227 # Register the inputs on each input sheet 228 generatorsheets = topo.sim.objects(GeneratorSheet) 229 if not isinstance(inputs,dict): 230 for g in generatorsheets.values(): 231 g.set_input_generator(inputs) 232 else: 233 for each in inputs.keys(): 234 if generatorsheets.has_key(each): 235 generatorsheets[each].set_input_generator(inputs[each]) 236 else: 237 param.Parameterized().warning( 238 '%s not a valid Sheet name for pattern_present.' % each) 239 240 topo.sim.event_push() 241 # CBENHANCEMENT: would be nice to break this up for visualizing motion 242 topo.sim.run(duration) 243 topo.sim.event_pop() 244 245 # turn sheets' plasticity and output_fn plasticity back on if we turned it off before 246 247 if not plastic: 248 for sheet in topo.sim.objects(Sheet).values(): 249 sheet.restore_plasticity_state() 250 251 if not apply_output_fns: 252 for each in topo.sim.objects(Sheet).values(): 253 each.apply_output_fns = True 254 255 256 if not overwrite_previous: 257 restore_input_generators()

258

259 260 261 -class _VersionPrinter(object):

262 """When unpickled, prints version & release information about snapshot."""

263 - def __init__(self,release,version):

264 self.release = release 265 self.version = version

266 - def __getstate__(self):

267 return {'release':self.release, 268 'version':self.version}

269 - def __setstate__(self,state):

270 release = state['release'] 271 version = state['version'] 272 self.release=release 273 self.version=version 274 param.Parameterized(name="load_snapshot").debug("Snapshot is from release '%s' (version '%s')."%(release,version))

275

276 277 -def save_snapshot(snapshot_name=None,xml=False):

278 """ 279 Save a snapshot of the network's current state. 280 281 The snapshot is saved as a gzip-compressed Python binary pickle. 282 283 (xml snapshots are currently experimental, and will not be useful 284 for most users.) 285 286 As this function uses Python's 'pickle' module, it is subject to 287 the same limitations (see the pickle module's documentation) - 288 with the notable exception of class attributes. Python does not 289 pickle class attributes, but this function stores class attributes 290 of any Parameterized class that is declared within the topo 291 package. See the param.parameterized.PicklableClassAttributes 292 class for more information. 293 """ 294 if not snapshot_name: 295 snapshot_name = topo.sim.basename() + ".typ" 296 297 # For now we just search topo, but could do same for other packages. 298 299 # CEBALERT: shouldn't it be topo and param? I guess we already get 300 # many classes defined in param because they are imported into 301 # topo at some point anyway. 302 topoPOclassattrs = PicklableClassAttributes(topo,exclusions=('plotting','tests','tkgui'), 303 startup_commands=topo.sim.startup_commands) 304 305 from topo.misc.commandline import global_params 306 307 topo.sim.RELEASE=topo.release 308 topo.sim.VERSION=topo.version 309 310 # CEBHACKALERT: is a tuple guaranteed to be unpacked in order? 311 # If not, then startup commands are not necessarily executed before 312 # the simulation is unpickled 313 # 314 # CB: if we first pickle.dumps() each of these things, then 315 # pickle.dump() a dictionary (probably), we'll have more control 316 # over unpickling. E.g. we could in the future choose not to 317 # unpickle something. And we can certainly control the unpickling 318 # order this way. 319 320 to_save = (_VersionPrinter(topo.release,topo.version),PickleMain(),global_params,topoPOclassattrs,topo.sim) 321 322 if not xml: 323 try: 324 snapshot_file=gzip.open(normalize_path(snapshot_name),'wb',compresslevel=5) 325 except NameError: 326 snapshot_file=open(normalize_path(snapshot_name),'wb') 327 328 pickle.dump(to_save,snapshot_file,2) 329 else: 330 snapshot_file=open(normalize_path(snapshot_name),'w') 331 332 try: 333 import gnosis.xml.pickle 334 gnosis.xml.pickle.dump(to_save,snapshot_file,2,allow_rawpickles=True) 335 except ImportError: 336 param.Parameterized().warning("Unable to import 'gnosis' module: no snapshot saved (xml snapshots unavailable).") 337 338 339 snapshot_file.close()

340

341 342 343 -def _load_pickle(snapshot):

344 345 try: 346 import gnosis.xml.pickle 347 gnosis.xml.pickle.load(snapshot,allow_rawpickles=True,class_search=gnosis.xml.pickle.SEARCH_ALL) 348 except ExpatError: # have gnosis.xml but snapshot is not xml 349 snapshot.seek(0) 350 pickle.load(snapshot) 351 except ImportError: # no gnosis.xml; snapshot could still be xml but there's nothing we can do 352 pickle.load(snapshot)

353

354 355 356 -def load_snapshot(snapshot_name):

357 """ 358 Load the simulation stored in snapshot_name. 359 """ 360 # unpickling the PicklableClassAttributes() executes startup_commands and 361 # sets PO class parameters. 362 363 snapshot_name = param.resolve_path(snapshot_name) 364 365 # If it's not gzipped, open as a normal file. 366 try: 367 snapshot = gzip.open(snapshot_name,'r') 368 snapshot.read(1) 369 snapshot.seek(0) 370 except (IOError,NameError): 371 snapshot = open(snapshot_name,'r') 372 373 374 try: 375 _load_pickle(snapshot) 376 except Exception, original_exception: 377 p = param.Parameterized(name="load_snapshot") 378 p.message("snapshot '%s' couldn't be loaded; installing legacy support"%snapshot_name) 379 import topo.misc.legacy as L 380 L.SnapshotSupport.install() 381 try: 382 _load_pickle(snapshot) 383 p.message("snapshot loaded successfully with legacy support") 384 except: 385 import traceback 386 m = """ 387 Snapshot could not be loaded. 388 389 If you make a copy of the snapshot available to 390 Topographica's developers, support for it can be added to 391 Topographica; please file a bug report via the website. 392 393 Original error: 394 395 %s 396 """%traceback.format_exc() 397 p.warning(m) 398 print "Error after loading legacy support:\n" 399 raise original_exception 400 401 snapshot.close() 402 403 # Restore subplotting prefs without worrying if there is a 404 # problem (e.g. if topo/analysis/ is not present) 405 try: 406 from topo.analysis.featureresponses import Subplotting 407 Subplotting.restore_subplots() 408 except: 409 p = param.Parameterized(name="load_snapshot") 410 p.message("Unable to restore Subplotting settings")

411

412 413 414 415 -def save_script_repr(script_name=None):

416 """ 417 Save the current simulation as a Topographica script. 418 419 Generates a script that, if run, would generate a simulation with 420 the same architecture as the one currently in memory. This can be 421 useful when defining networks in place, so that the same general 422 configuration can be recreated later. It also helps when 423 comparing two similar networks generated with different scripts, 424 so that the corresponding items can be matched rigorously. 425 426 Note that the result of this operation is usually just a starting 427 point for further editing, because it will not usually be runnable 428 as-is (for instance, some parameters may not have runnable 429 representations). Even so, this is usually a good start. 430 """ 431 if not script_name: 432 script_name = topo.sim.basename() + "_script_repr.ty" 433 434 header = ("# Generated by Topographica %s on %s\n\n" % 435 (topo.release,time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime()))) 436 script = header+topo.sim.script_repr() 437 438 script_file = open(normalize_path(script_name),'w') 439 script_file.write(script)

440 441 442 # Location of the version-controlled topographica directory (i.e. path 443 # of topo/ but up a level). Could be None. Nothing should assume that 444 # there is a version control system available. 445 try: 446 vc_topographica_dir = os.path.split(os.path.split(topo.__file__)[0])[0] 447 except: 448 vc_topographica_dir = None

449 450 451 # decorator that changes to vc_topographica_dir for duration of fn, 452 # if there is such a directory. Otherwise, doesn't change directory. 453 -def in_vc_topographica_dir(fn):

454 import os 455 def temporarily_change_to_vc_topographica_dir(*args,**kw): 456 orig_path = os.getcwd() 457 if vc_topographica_dir is not None: 458 os.chdir(vc_topographica_dir) 459 try: 460 result = fn(*args,**kw) 461 finally: 462 # ensure dir put back even if there's an error calling fn 463 if os.getcwd()!=orig_path: 464 os.chdir(orig_path) 465 return result

466 return temporarily_change_to_vc_topographica_dir 467

468 469 @in_vc_topographica_dir 470 -def _get_vc_commands():

471 # return name of version control system (None if no vc could be 472 # detected) 473 import os.path 474 vc_types = {'git':["status","diff",["log","-n1"],["svn","log","--limit=1"]], 475 'svn':["info","status","diff"], 476 'bzr':['info','status','diff']} 477 for vc_type,commands in vc_types.items(): 478 if os.path.exists(".%s"%vc_type): 479 return vc_type,commands

480

481 @in_vc_topographica_dir 482 -def _print_vc_info(filename):

483 """Save the version control status of the current code to the specified file.""" 484 try: 485 import subprocess 486 file = open(normalize_path(filename),'w') 487 file.write("Information about working copy used for batch run\n\n") 488 file.write("topo.version=%s\n"% topo.version) 489 file.flush() 490 vctype,commands = _get_vc_commands() 491 for cmd in commands: 492 fullcmd = [vctype,cmd] if isinstance(cmd,str) else [vctype]+cmd 493 494 # Note that we do not wait for the process below to finish 495 # (by calling e.g. wait() on the Popen object). Although 496 # this was probably done unintentionally, for a slow svn 497 # connection, it's an advantage. But it does mean the 498 # output of each command can appear in the file at any 499 # time (i.e. the command outputs appear in the order of 500 # finishing, rather than in the order of starting, making 501 # it impossible to label the commands). 502 subprocess.Popen(fullcmd,stdout=file,stderr=subprocess.STDOUT) 503 except: 504 print "Unable to retrieve version control information." 505 finally: 506 file.close()

507

508 509 -def _save_parameters(p,filename):

510 from topo.misc.commandline import global_params 511 512 g = {'global_params_specified':p, 513 'global_params_all':dict(global_params.get_param_values())} 514 515 for d in g.values(): 516 if 'name' in d: 517 del d['name'] 518 if 'print_level' in d: 519 del d['print_level'] 520 521 pickle.dump(g,open(normalize_path(filename),'w'))

522

523 524 # I'd expect your personal name_replacements to be set in some file 525 # you use to create batch runs, but it can alsp be set on the 526 # commandline. Before calling run_batch(), include something like the 527 # following: 528 # run_batch.dirname_params_filter.map=OrderedDict(("cortex_density","cd")) 529 530 -class param_formatter(ParameterizedFunction):

531 532 # CEBALERT: should I have made this a parameter at the run_batch 533 # level? And I don't know what to call it. 534 map = param.Dict(default=OrderedDict(),doc=""" 535 Optional ordered dictionary of alternative names to use for 536 parameters, parameter_name:alternative_name 537 538 Use to shorten common parameter names (directory names are 539 limited in length on most file systems), and to specify an 540 order. 541 542 Names not specified here will be sorted alphabetically.""") 543

544 - def __call__(self,params):

545 result = "" 546 # present in params but not in map 547 unspecified_in_map = sorted(set(params).difference(set(self.map))) 548 # present in params and in map, preserving order of map 549 specified_in_map = [n for n in self.map.keys() if n in params] 550 551 for pname in specified_in_map+unspecified_in_map: 552 val = params[pname] 553 # Special case to give reasonable filenames for lists 554 valstr= ("_".join([str(i) for i in val]) if isinstance(val,list) 555 else str(val)) 556 result += "," + self.map.get(pname,pname) + "=" + valstr 557 return result

558 559 560 # Used only by default_analysis_function 561 # Should be in order they are needed; e.g. Activity after map measurement, 562 # in case Activity plot includes map subplots 563 default_analysis_plotgroups=["Orientation Preference","Activity"]

564 565 -def default_analysis_function():

566 """ 567 Basic example of an analysis command for run_batch; users are 568 likely to need something similar but highly customized. 569 """ 570 # CEBALERT: why are these imports here rather than at the top? 571 import topo 572 from topo.command.analysis import save_plotgroup 573 from topo.base.projection import ProjectionSheet 574 575 # Save all plotgroups listed in default_analysis_plotgroups 576 for pg in default_analysis_plotgroups: 577 save_plotgroup(pg,use_cached_results=True) 578 579 # Plot projections from each measured map 580 measured_sheets = [s for s in topo.sim.objects(ProjectionSheet).values() 581 if hasattr(s,'measure_maps') and s.measure_maps] 582 for s in measured_sheets: 583 for p in s.in_connections: 584 save_plotgroup("Projection",projection=p) 585 586 # Test response to a standardized pattern 587 from topo.pattern.basic import Gaussian 588 from math import pi 589 pattern_present(inputs=Gaussian(orientation=pi/4,aspect_ratio=4.7)) 590 save_plotgroup("Activity",saver_params={"filename_suffix":"_45d"})

591

592 593 # ALERT: Need to move docs into params. 594 -class run_batch(ParameterizedFunction):

595 """ 596 Run a Topographica simulation in batch mode. 597 598 Features: 599 600 - Generates a unique, well-defined name for each 'experiment' 601 (i.e. simulation run) based on the date, script file, and 602 parameter settings. Note that very long names may be truncated 603 (see the max_name_length parameter). 604 605 - Allows parameters to be varied on the command-line, 606 to allow comparing various settings 607 608 - Saves a script capturing the simulation state periodically, 609 to preserve parameter values from old experiments and to allow 610 them to be reproduced exactly later 611 612 - Can perform user-specified analysis routines periodically, 613 to monitor the simulation as it progresses. 614 615 - Stores commandline output (stdout) in the output directory 616 617 A typical use of this function is for remote execution of a large 618 number of simulations with different parameters, often on remote 619 machines (such as clusters). 620 621 The script_file parameter defines the .ty script we want to run in 622 batch mode. The output_directory defines the root directory in 623 which a unique individual directory will be created for this 624 particular run. The optional analysis_fn can be any python 625 function to be called at each of the simulation iterations defined 626 in the analysis times list. The analysis_fn should perform 627 whatever analysis of the simulation you want to perform, such as 628 plotting or calculating some statistics. The analysis_fn should 629 avoid using any GUI functions (i.e., should not import anything 630 from topo.tkgui), and it should save all of its results into 631 files. 632 633 As a special case, a number can be passed for the times list, in 634 which case it is used to scale a default list of times up to 635 10000; e.g. times=2 will select a default list of times up to 636 20000. Alternatively, an explicit list of times can be supplied. 637 638 Any other optional parameters supplied will be set in the main 639 namespace before any scripts are run. They will also be used to 640 construct a unique topo.sim.name for the file, and they will be 641 encoded into the simulation directory name, to make it clear how 642 each simulation differs from the others. 643 644 If requested by setting snapshot=True, saves a snapshot at the 645 end of the simulation. 646 647 If available and requested by setting vc_info=True, prints 648 the revision number and any outstanding diffs from the version 649 control system. 650 651 Note that this function alters param.normalize_path.prefix so that 652 all output goes into the same location. The original value of 653 param.normalize_path.prefix is deliberately not restored at the 654 end of the function so that the output of any subsequent commands 655 will go into the same place. 656 """ 657 output_directory=param.String("Output") 658 659 analysis_fn = param.Callable(default_analysis_function) 660 661 times = param.Parameter(1.0) 662 663 snapshot=param.Boolean(True) 664 665 vc_info=param.Boolean(True) 666 667 dirname_prefix = param.String(default="",doc=""" 668 Optional prefix for the directory name (allowing e.g. easy 669 grouping).""") 670 671 # CB: do any platforms also have a maximum total path length? 672 max_name_length = param.Number(default=200,doc=""" 673 The experiment's directory name will be truncated at this 674 number of characters (since most filesystems have a 675 limit).""") 676 677 name_time_format = param.String(default="%Y%m%d%H%M",doc=""" 678 String format for the time included in the output directory 679 and file names. See the Python time module library 680 documentation for codes. 681 682 E.g. Adding '%S' to the default would include seconds.""") 683 684 save_global_params = param.Boolean(default=True,doc=""" 685 Whether to save the script's global_parameters to a pickle in 686 the output_directory after the script has been loaded (for 687 e.g. future inspection of the experiment).""") 688 689 dirname_params_filter = param.Callable(param_formatter.instance(),doc=""" 690 Function to control how the parameter names will appear in the 691 output_directory's name.""") 692 693

694 - def _truncate(self,p,s):

695 """ 696 If s is greater than the max_name_length parameter, truncate it 697 (and indicate that it has been truncated). 698 """ 699 # '___' at the end is supposed to represent '...' 700 return s if len(s)<=p.max_name_length else s[0:p.max_name_length-3]+'___'

701

702 - def __call__(self,script_file,**params_to_override):

703 p=ParamOverrides(self,params_to_override,allow_extra_keywords=True) 704 705 import os 706 import shutil 707 708 # Construct simulation name, etc. 709 scriptbase= re.sub('.ty$','',os.path.basename(script_file)) 710 prefix = "" 711 prefix += time.strftime(p.name_time_format) 712 prefix += "_" + scriptbase 713 simname = prefix 714 715 # Construct parameter-value portion of filename; should do more filtering 716 # CBENHANCEMENT: should provide chance for user to specify a 717 # function (i.e. make this a function, and have a parameter to 718 # allow the function to be overridden). 719 # And sort by name by default? Skip ones that aren't different 720 # from default, or at least put them at the end? 721 prefix += p.dirname_params_filter(p.extra_keywords()) 722 723 # Set provided parameter values in main namespace 724 from topo.misc.commandline import global_params 725 global_params.set_in_context(**p.extra_keywords()) 726 727 # Create output directories 728 if not os.path.isdir(normalize_path(p['output_directory'])): 729 os.mkdir(normalize_path(p['output_directory'])) 730 731 732 dirname = self._truncate(p,p.dirname_prefix+prefix) 733 normalize_path.prefix = normalize_path(os.path.join(p['output_directory'],dirname)) 734 735 if os.path.isdir(normalize_path.prefix): 736 print "Batch run: Warning -- directory already exists!" 737 print "Run aborted; wait one minute before trying again, or else rename existing directory: \n" + \ 738 normalize_path.prefix 739 740 sys.exit(-1) 741 else: 742 os.mkdir(normalize_path.prefix) 743 print "Batch run output will be in " + normalize_path.prefix 744 745 746 if p['vc_info']: 747 _print_vc_info(simname+".diffs") 748 749 hostinfo = "Host: " + " ".join(platform.uname()) 750 topographicalocation = "Topographica: " + os.path.abspath(sys.argv[0]) 751 topolocation = "topo package: " + os.path.abspath(topo.__file__) 752 scriptlocation = "script: " + os.path.abspath(script_file) 753 754 starttime=time.time() 755 startnote = "Batch run started at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000", 756 time.gmtime()) 757 command_used_to_start = string.join(sys.argv) 758 759 # Shadow stdout to a .out file in the output directory, so that 760 # print statements will go to both the file and to stdout. 761 batch_output = open(normalize_path(simname+".out"),'w') 762 batch_output.write(command_used_to_start+"\n") 763 sys.stdout = MultiFile(batch_output,sys.stdout) 764 765 print 766 print hostinfo 767 print topographicalocation 768 print topolocation 769 print scriptlocation 770 print 771 print startnote 772 773 from topo.misc.commandline import auto_import_commands 774 auto_import_commands() 775 776 # Ensure that saved state includes all parameter values 777 from topo.command.basic import save_script_repr 778 param.parameterized.script_repr_suppress_defaults=False 779 780 # Save a copy of the script file for reference 781 shutil.copy2(script_file, normalize_path.prefix) 782 shutil.move(normalize_path(scriptbase+".ty"), 783 normalize_path(simname+".ty")) 784 785 786 # Default case: times is just a number that scales a standard list of times 787 times=p['times'] 788 if not isinstance(times,list): 789 times=[t*times for t in [0,50,100,500,1000,2000,3000,4000,5000,10000]] 790 791 # Run script in main 792 error_count = 0 793 initial_warning_count = param.parameterized.warning_count 794 try: 795 execfile(script_file,__main__.__dict__) #global_params.context 796 global_params.check_for_unused_names() 797 if p.save_global_params: 798 _save_parameters(p.extra_keywords(),simname+".global_params.pickle") 799 print_sizes() 800 topo.sim.name=simname 801 802 # Run each segment, doing the analysis and saving the script state each time 803 for run_to in times: 804 topo.sim.run(run_to - topo.sim.time()) 805 p['analysis_fn']() 806 save_script_repr() 807 elapsedtime=time.time()-starttime 808 param.Parameterized(name="run_batch").message( 809 "Elapsed real time %02d:%02d." % (int(elapsedtime/60),int(elapsedtime%60))) 810 811 if p['snapshot']: 812 save_snapshot() 813 814 except: 815 error_count+=1 816 import traceback 817 traceback.print_exc(file=sys.stdout) 818 sys.stderr.write("Warning -- Error detected: execution halted.\n") 819 820 821 print "\nBatch run completed at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000", 822 time.gmtime()) 823 print "There were %d error(s) and %d warning(s)%s." % \ 824 (error_count,(param.parameterized.warning_count-initial_warning_count), 825 ((" (plus %d warning(s) prior to entering run_batch)"%initial_warning_count 826 if initial_warning_count>0 else ""))) 827 828 # restore stdout 829 sys.stdout = sys.__stdout__ 830 batch_output.close()

831

832 833 834 -def wipe_out_activity():

835 """ 836 Resets activity of all Sheets and their connections to zero. 837 """ 838 # ALERT: this works for now, but it may need to be implemented 839 # recursively using methods implemented separately on each class, 840 # if there are often new types of objects created that store an 841 # activity value. 842 for s in topo.sim.objects(Sheet).values(): 843 s.activity*=0.0 844 for c in s.in_connections: 845 if hasattr(c,'activity'): 846 c.activity*=0.0

847

848 849 850 -def n_bytes():

851 """ 852 Estimate the minimum memory needed for the Sheets in this Simulation, in bytes. 853 854 This estimate is a lower bound only, based primarily on memory for 855 the matrices used for activity and connections. 856 """ 857 return sum([s.n_bytes() for s in topo.sim.objects(Sheet).values()])

858

859 860 861 -def n_conns():

862 """ 863 Count the number of connections in all ProjectionSheets in the current Simulation. 864 """ 865 return sum([s.n_conns() for s in topo.sim.objects(ProjectionSheet).values()])

866

867 868 -def print_sizes():

869 """Format the results from n_conns() and n_bytes() for use in batch output.""" 870 print "Defined %d-connection network; %0.0fMB required for weight storage." % \ 871 (n_conns(),max(n_bytes()/1024.0/1024.0,1.0))

872 873 # added these two function to the PatternDrivenAnalysis hooks 874 PatternDrivenAnalysis.pre_presentation_hooks.append(wipe_out_activity) 875 PatternDrivenAnalysis.pre_presentation_hooks.append(clear_event_queue) 876 877 878 # maybe an explicit list would be better? 879 import types 880 __all__ = list(set([k for k,v in locals().items() 881 if isinstance(v,types.FunctionType) or 882 (isinstance(v,type) and issubclass(v,ParameterizedFunction)) 883 and not v.__name__.startswith('_')])) 884

Source Code for Module topo.command.basic