Source Code for Module topo.misc.memuse

  1  """ 
  2  Functions for measuring memory usage, to allow usage to be optimized. 
  3   
  4  Examples:: 
  5   
  6    bin/python -c 'execfile("topo/misc/memuse.py") ; print topsize_mb()' 
  7     
  8    ./topographica -c 'from topo.misc import memuse' -c 'print memuse.topsize_mb()' 
  9     
 10    ./topographica -c 'from topo.misc import memuse, asizeof' -c 'print memuse.topsize_mb()' 
 11     
 12    ./topographica -a -c 'from topo.misc import memuse, asizeof' -c 'print memuse.topsize_mb()' 
 13     
 14    ./topographica -a -c 'from topo.misc import memuse, asizeof' examples/tiny.ty -c 'print memuse.allsizes_mb()' 
 15     
 16    ./topographica -a -c 'from topo.misc import memuse, asizeof' -c 'memuse.memuse_batch("examples/tiny.ty",cortex_density=20)' 
 17   
 18    ./topographica -a -c 'from topo.misc import memuse, asizeof' -c 'memuse.memuse_batch("examples/tiny.ty",times=[0,100],analysis_fn=memuse.plotting_and_saving_analysis_fn,cortex_density=20)' 
 19   
 20  $Id: memuse.py 11326 2010-07-30 15:42:45Z jbednar $ 
 21  """ 
 22  __version__='$Revision: 10367 $' 
 23   
 24  # If functions in this file need anything other than the very basic 
 25  # imports declared here at the top, they must do so using import 
 26  # statements *within* the function definition to avoid polluting 
 27  # memory.  Otherwise, the simplest functions in this file would end up 
 28  # measuring memory taken by unused and irrelevant imports like 'topo', 
 29  # which doesn't ever need to be loaded for the 'execfile' example 
 30  # shown above. 
 31   
 32  import subprocess 
 33   
 34   
 35 -def cmd_to_string(cmd): 
 36      """Run a system command as in os.system(), but capture the stdout and return it as a string.""" 
 37      return subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE).communicate()[0] 
 38   
 39   
 40   
 41 -def topsize(): 
 42      """Return the RES size of this process as reported by the top(1) command.""" 
 43      import os 
 44      top_line=cmd_to_string("top -n 1 -b -p %d | grep '^[ ]*%d '" % (os.getpid(),os.getpid())) 
 45      return top_line.split()[5] 
 46   
 47   
 48   
 49 -def simsize(): 
 50      """ 
 51      Return the size of topo.sim reported by asizeof.asizeof(). 
 52      This estimate does not currently include any numpy arrays, and 
 53      may also be missing other important items. 
 54   
 55      Python 2.6 supports getsizeof() and a __sizeof__ attribute that user 
 56      code can implement, which should provide a more accurate estimate. 
 57      """ 
 58      import asizeof,topo 
 59      return asizeof.asizeof(topo.sim) 
 60   
 61   
 62   
 63  ############################################################################### 
 64  # String-formatted versions of the above 
 65   
 66 -def mb(bytes): 
 67      """Format the given value in bytes as a string in megabytes""" 
 68      return "%dMB" % (bytes/1024.0/1024.0) 
 69   
 70 -def topsize_mb(): 
 71      """String-formatted version of the RES size of this process reported by top(1).""" 
 72      return "topsize:%s" % topsize() 
 73   
 74 -def simsize_mb(): 
 75      """String-formatted version of the value reported by asizeof(topo.sim).""" 
 76      return "simsize:%s" % (mb(simsize())) 
 77   
 78 -def wtsize_mb(): 
 79      """String-formatted version of the memory taken by the weights, from print_sizes().""" 
 80      from topo.command.basic import n_bytes 
 81      return "wtsize:%s" % (mb(n_bytes())) 
 82   
 83 -def allsizes_mb(): 
 84      """ 
 85      Collates results from topsize, simsize, and wtsize. 
 86   
 87      Formatted to suggest that the topsize is made up of code (not 
 88      currently estimated), topo.sim (apart from weights), and weights. 
 89      """ 
 90      from topo.command.basic import n_bytes 
 91      return "%s =? code + %s + %s (%s tot)" % (topsize_mb(),simsize_mb(),wtsize_mb(),mb(simsize()+n_bytes())) 
 92   
 93   
 94  ############################################################################### 
 95  # Batch commands 
 96   
 97   
 98 -def default_memuse_analysis_fn(prefix=""): 
 99      """Basic memuse function for use with memuse_batch()""" 
100      import topo 
101      print "%st%s: %s" % (prefix,topo.sim.timestr(),allsizes_mb()) 
102   
103   
104   
105 -def plotting_and_saving_analysis_fn(prefix=""): 
106      """For use with memuse_batch() to test snapshot and plotting memory usage.""" 
107      import topo 
108      from topo.command.basic import save_snapshot 
109      from topo.command.analysis import measure_sine_pref,save_plotgroup 
110       
111      print "%sMemuse at time %s: %s" % (prefix,topo.sim.timestr(),allsizes_mb()) 
112      measure_sine_pref() 
113      print "%sAfter measure_sine_pref:  %s" % (prefix,allsizes_mb()) 
114      save_plotgroup("Orientation Preference") 
115      print "%sAfter save_plotgroup:     %s" % (prefix,allsizes_mb()) 
116      save_snapshot("/tmp/tmp.typ") 
117      print "%sAfter save_snapshot:      %s" % (prefix,allsizes_mb()) 
118   
119   
120   
121 -def memuse_batch(script_file,times=[0],analysis_fn=default_memuse_analysis_fn,**params): 
122      """ 
123      Similar to run_batch, but analyzes the memory requirement of a simulation at different times. 
124       
125      First, the specified script file will be run using the specified parameters. 
126      Then at each of the specified times, the given analysis_fn (which 
127      calls allsizes_mb() by default) is run.  The output is labeled 
128      with the script file, time, and parameters so that results from 
129      different runs can be compared. 
130      """ 
131      import os,re,__main__,topo 
132      from topo.misc.commandline import global_params 
133   
134      # Construct simulation name, etc. 
135      scriptbase= re.sub('.ty$','',os.path.basename(script_file)) 
136      prefix = "" 
137      #prefix += time.strftime("%Y%m%d%H%M") + "_" 
138      prefix += scriptbase 
139      simname = prefix 
140       
141      # Construct parameter-value portion of filename; should do more filtering 
142      for a,val in params.iteritems(): 
143          # Special case to give reasonable filenames for lists 
144          valstr= ("_".join([str(i) for i in val]) if isinstance(val,list) 
145                   else str(val)) 
146          prefix += "," + a + "=" + valstr 
147   
148      # Set provided parameter values in main namespace 
149      global_params.set_in_context(**params) 
150       
151      # Run script in main 
152      try: 
153          execfile(script_file,__main__.__dict__) #global_params.context 
154          global_params.check_for_unused_names() 
155          topo.sim.name=simname 
156       
157          # Run each segment, doing the analysis and saving the script state each time 
158          for run_to in times: 
159              topo.sim.run(run_to - topo.sim.time()) 
160              analysis_fn(prefix=prefix+": ") 
161   
162      except: 
163          import traceback,sys 
164          traceback.print_exc(file=sys.stdout) 
165          sys.stderr.write("Warning -- Error detected: execution halted.\n") 
166