yade-dev team mailing list archive
-
yade-dev team
-
Mailing list archive
-
Message #00748
[svn] r1562 - in trunk: core gui gui/py scripts
Author: eudoxos
Date: 2008-10-28 11:30:53 +0100 (Tue, 28 Oct 2008)
New Revision: 1562
Added:
trunk/gui/py/yade-multi
trunk/scripts/multi.py
trunk/scripts/multi.table
Modified:
trunk/core/MetaBody.cpp
trunk/gui/SConscript
trunk/gui/py/PythonUI_rc.py
trunk/gui/py/utils.py
Log:
1. Commit simple parametric study interface, documented at http://yade.wikia.com/index.php?title=ScriptParametricStudy and wrapper script to queue jobs, which is installed as yade-trunk-multi, yade-trunk-opt-multi and so on.
2. Examples of such parametric "study" added in scripts/multi.py and scripts/multi.table
3. MetaBody now create Omega().tags['id'] at initialization like '20081028T102950p15498' (date, time, pid) that is unique
Modified: trunk/core/MetaBody.cpp
===================================================================
--- trunk/core/MetaBody.cpp 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/core/MetaBody.cpp 2008-10-28 10:30:53 UTC (rev 1562)
@@ -52,6 +52,7 @@
string gecos(pw->pw_gecos), gecos2; size_t p=gecos.find(","); if(p!=string::npos) boost::algorithm::erase_tail(gecos,gecos.size()-p); for(size_t i=0;i<gecos.size();i++){gecos2.push_back(((unsigned char)gecos[i])<128 ? gecos[i] : '?'); }
tags.push_back(boost::algorithm::replace_all_copy(string("author=")+gecos2+" ("+string(pw->pw_name)+"@"+hostname+")"," ","~"));
tags.push_back(string("isoTime="+boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())));
+ tags.push_back(string("id="+boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())+"p"+lexical_cast<string>(getpid())));
tags.push_back(string("description="));
}
Modified: trunk/gui/SConscript
===================================================================
--- trunk/gui/SConscript 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/gui/SConscript 2008-10-28 10:30:53 UTC (rev 1562)
@@ -52,6 +52,8 @@
env.File('qt.py','qt3'),
])
+ env.InstallAs('$PREFIX/bin/yade$SUFFIX-multi',env.File('yade-multi','py'))
+
# python modules are one level deeper so that you can say: from yade.wrapper import *
env.Install('$PREFIX/lib/yade$SUFFIX/gui/yade',[
env.SharedLibrary('wrapper',['py/yadeControl.cpp'],SHLIBPREFIX='',
Modified: trunk/gui/py/PythonUI_rc.py
===================================================================
--- trunk/gui/py/PythonUI_rc.py 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/gui/py/PythonUI_rc.py 2008-10-28 10:30:53 UTC (rev 1562)
@@ -24,6 +24,7 @@
import yade.PythonTCPServer
srv=yade.PythonTCPServer.PythonTCPServer(minPort=9000)
yade.runtime.cookie=srv.server.cookie
+sys.stdout.flush()
## run simulation if requested from the command line
if runtime.simulation:
@@ -40,7 +41,9 @@
import traceback
traceback.print_exc()
if(runtime.nonInteractive or runtime.stopAfter): sys.exit(1)
-if runtime.stopAfter: sys.exit(0)
+if runtime.stopAfter:
+ print "Finished, bye."
+ sys.exit(0)
# run commands if requested from the command line
#if yadeRunCommands:
Modified: trunk/gui/py/utils.py
===================================================================
--- trunk/gui/py/utils.py 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/gui/py/utils.py 2008-10-28 10:30:53 UTC (rev 1562)
@@ -16,7 +16,7 @@
# c++ implementations for performance reasons
from yade._utils import *
-def saveVars(**kw):
+def saveVars(mark='',**kw):
"""Save passed variables into the simulation so that it can be recovered when the simulation is loaded again.
For example, variables a=5, b=66 and c=7.5e-4 are defined. To save those, use
@@ -31,12 +31,12 @@
and they will be defined in the __builtin__ namespace (i.e. available from anywhere in the python code).
"""
import cPickle
- Omega().tags['pickledPythonVariablesDictionary']=cPickle.dumps(kw)
+ Omega().tags['pickledPythonVariablesDictionary'+mark]=cPickle.dumps(kw)
-def loadVars():
+def loadVars(mark=''):
import cPickle
import __builtin__
- d=cPickle.loads(Omega().tags['pickledPythonVariablesDictionary'])
+ d=cPickle.loads(Omega().tags['pickledPythonVariablesDictionary'+mark])
for k in d: __builtin__.__dict__[k]=d[k]
@@ -241,14 +241,14 @@
mainloop.run()
pipeline.set_state(gst.STATE_NULL); pipeline.get_state()
-def readParamsFromTable(tableFile=None,tableLine=None,noTableOk=False,**kw):
+def readParamsFromTable(tableFileLine=None,noTableOk=False,**kw):
"""
Read parameters from a file and assign them to __builtin__ variables.
tableFile is a text file (with one value per blank-separated columns)
tableLine is number of line where to get the values from
- The format of the file is as follows (no comments, empty lines etc allowed…)
+ The format of the file is as follows (commens starting with # and empty lines allowed
name1 name2 … # 0th line
val1 val2 … # 1st line
@@ -261,17 +261,22 @@
assigns Omega().tags['defaultParams']="unassignedName1=defaultValue1,…"
+ saves all parameters (default as well as settable) using saveVars('table')
+
return value is the number of assigned parameters.
"""
o=Omega()
tagsParams=[]
+ dictDefaults,dictParams={},{}
import os, __builtin__
- if not tableFile and not os.environ.has_key('PARAM_TABLE'):
+ if not tableFileLine and not os.environ.has_key('PARAM_TABLE'):
if not noTableOk: raise EnvironmentError("PARAM_TABLE is not defined in the environment")
+ o.tags['line']='l!'
else:
- if not tableFile: tableFile=os.environ['PARAM_TABLE']
- if not tableLine: tableLine=int(os.environ['PARAM_LINE'])
- ll=open(tableFile).readlines(); names=ll[0].split(); values=ll[tableLine].split()
+ if not tableFileLine: tableFileLine=os.environ['PARAM_TABLE']
+ tableFile,tableLine=tableFileLine.split(':')
+ o.tags['line']='l'+tableLine
+ ll=[l.split('#')[0] for l in open(tableFile).readlines()]; names=ll[0].split(); values=ll[int(tableLine)].split()
assert(len(names)==len(values))
for i in range(len(names)):
if names[i]=='description': o.tags['description']=values[i]
@@ -279,12 +284,13 @@
if names[i] not in kw.keys(): raise NameError("Parameter `%s' has no default value assigned"%names[i])
kw.pop(names[i])
eq="%s=%s"%(names[i],values[i])
- exec('__builtin__.%s=%s'%(names[i],values[i])); tagsParams+=['%s=%s'%(names[i],values[i])]
+ exec('__builtin__.%s=%s'%(names[i],values[i])); tagsParams+=['%s=%s'%(names[i],values[i])]; dictParams[names[i]]=values[i]
defaults=[]
for k in kw.keys():
exec("__builtin__.%s=%s"%(k,kw[k]))
- defaults+=["%s=%s"%(k,kw[k])]
+ defaults+=["%s=%s"%(k,kw[k])]; dictDefaults[k]=kw[k]
o.tags['defaultParams']=",".join(defaults)
o.tags['params']=",".join(tagsParams)
+ dictParams.update(dictDefaults); saveVars('table',**dictParams)
return len(tagsParams)
Added: trunk/gui/py/yade-multi
===================================================================
--- trunk/gui/py/yade-multi 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/gui/py/yade-multi 2008-10-28 10:30:53 UTC (rev 1562)
@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# portions © 2008 Václav Šmilauer <eudoxos@xxxxxxxx>
+
+
+#
+# _MANY_ thanks to Mark Pettit for concurrent jobs handling!
+# http://code.activestate.com/recipes/534160/
+#
+
+import os, sys, thread, time
+
+def __concurrent_batch(cmd,thread_num,completion_status_dict,exit_code_dict):
+ """Helper routine for 'concurrent_batches."""
+
+ t0=time.time()
+ print '#%d started on %s'%(thread_num,time.asctime())
+ exit_code_dict[thread_num] = os.system(cmd)
+ completion_status_dict[thread_num] = 1 # for sum() routine
+ dt=int(time.time()-t0)
+ strDt='%02d:%02d:%02d'%(dt//3600,(dt%3600)//60,(dt%60))
+ strStatus='done ' if exit_code_dict[thread_num]==0 else 'FAILED '
+ print "#%d %s (exit status %d), duration %s"%(thread_num,strStatus,exit_code_dict[thread_num],strDt)
+
+def concurrent_batches(batchlist,maxjobs=0,maxtime=0):
+ """Run a list of batch commands simultaneously.
+
+ 'batchlist' is a list of strings suitable for submitting under os.system().
+ Each job will run in a separate thread, with the thread ending when the
+ subprocess ends.
+ 'maxjobs' will, if greater then zero, be the maximum number of simultaneous
+ jobs which can concurrently run. This would be used to limit the number of
+ processes where too many could flood a system, causing performance issues.
+ 'maxtime', when greater than zero, be the maximum amount of time (seconds)
+ that we will wait for processes to complete. After that, we will return,
+ but no jobs will be killed. In other words, the jobs still running will
+ continue to run, and hopefully finish in the course of time.
+
+ example: concurrent_batches(("gzip abc","gzip def","gzip xyz"))
+
+ returns: a dictionary of exit status codes, but only when ALL jobs are
+ complete, or the maximum time has been exceeded.
+ Note that if returning due to exceeding time, the dictionary will
+ continue to be updated by the threads as they complete.
+ The key of the dictionary is the thread number, which matches the
+ index of the list of batch commands. The value is the result of
+ the os.system call.
+
+ gotcha: If both the maxjobs and maxtime is set, there is a possibility that
+ not all jobs will be submitted. The only way to detect this will be
+ by checking for the absence of the KEY in the returned dictionary.
+ """
+
+ if not batchlist: return {}
+ completion_status_dict, exit_code_dict = {}, {}
+ num_jobs = len(batchlist)
+ start_time = time.time()
+ for thread_num, cmd in enumerate(batchlist):
+ exit_code_dict[thread_num] = None
+ completion_status_dict[thread_num] = 0 # for sum() routine
+ thread.start_new_thread(__concurrent_batch,
+ (cmd,thread_num,completion_status_dict,exit_code_dict))
+ while True:
+ completed = sum(completion_status_dict.values())
+ if num_jobs == completed:
+ return exit_code_dict # all done
+ running = thread_num - completed + 1
+ if maxtime > 0:
+ if time.time() - start_time > maxtime:
+ return exit_code_dict
+ if not maxjobs:
+ if thread_num < num_jobs-1: # have we submitted all jobs ?
+ break # no, so break to for cmd loop
+ else:
+ time.sleep(.2) # yes, so wait until jobs are complete
+ continue
+ if running < maxjobs and thread_num < num_jobs-1:
+ break # for next for loop
+ time.sleep(.2)
+#
+# now begins the yade code
+#
+
+import sys,re,optparse
+def getNumCores(): return len([l for l in open('/proc/cpuinfo','r') if l.find('processor')==0])
+
+parser=optparse.OptionParser(usage='%prog [options] TABLE SIMULATION.py\n\n %prog runs yade simulation multiple times with different parameters.\n See http://yade.wikia.com/wiki/ScriptParametricStudy for details.')
+parser.add_option('-j',dest='maxJobs',type='int',help="Maximum number of simultaneous jobs to run (default: number of cores, i.e. %d)"%getNumCores(),metavar='NUM',default=getNumCores())
+parser.add_option('--log',dest='logFormat',help='Format of log files -- must contain a %, which will be replaced by line number (default: SIMULATION.%.log)',metavar='FORMAT')
+parser.add_option('-l','--lines',dest='lineList',help='Lines of TABLE to use, in the format 2,3-5,8,11-13 (default: all available lines in TABLE)',metavar='LIST')
+parser.add_option('--nice',dest='nice',type='int',help='Nice value of spawned jobs (default: 10)',default=10)
+parser.add_option('--executable',dest='executable',help='Name of the program to run (default: %s)'%sys.argv[0][:-6],default=sys.argv[0][:-6],metavar='FILE') ## strip the '-multi' extension
+opts,args=parser.parse_args()
+logFormat,lineList,maxJobs,nice,executable=opts.logFormat,opts.lineList,opts.maxJobs,opts.nice,opts.executable
+
+if len(args)!=2:
+ #print "Exactly two non-option arguments must be specified -- parameter table and script to be run.\n"
+ parser.print_help()
+ sys.exit(1)
+table,simul=args[0:2]
+if not logFormat: logFormat=(simul[:-3] if simul[-3:]=='.py' else simul)+".%.log"
+if not '%' in logFormat: raise StandardError("Log string must contain a `%'")
+
+print "Will run `%s' on `%s' with nice value %d, output redirected to `%s', %d jobs at a time."%(executable,simul,nice,logFormat,maxJobs)
+
+ll=open(table,'r').readlines()
+availableLines=[i for i in range(len(ll)) if not re.match(r'^\s*(#.*)?$',ll[i][:-1]) and i>0]
+
+print "Will use table `%s', with available lines"%(table),', '.join([str(i) for i in availableLines])+'.'
+
+if lineList:
+ useLines=[]
+ def numRange2List(s):
+ ret=[]
+ for l in s.split(','):
+ if "-" in l: ret+=range(*[int(s) for s in l.split('-')]); ret+=[ret[-1]+1]
+ else: ret+=[int(l)]
+ return ret
+ useLines0=numRange2List(lineList)
+ for l in useLines0:
+ if l not in availableLines: print "WARNING: skipping unavailable line %d that was requested from the command line."%l
+ elif l==0: print "WARNING: skipping line 0 that should contain variable labels"
+ else: useLines+=[l]
+else: useLines=availableLines
+print "Will use lines ",', '.join([str(i) for i in useLines])+'.'
+
+
+batches=[]
+for l in useLines:
+ logFile=logFormat.replace('%',str(l))
+ batches.append('PARAM_TABLE=%s:%d nice -n %d %s -N PythonUI -- -n -x %s > %s 2>&1'%(table,l,nice,executable,simul,logFile))
+print "Job summary:"
+for i in range(len(batches)):
+ print ' #%d:'%i, batches[i]
+# OK, go now
+concurrent_batches(batches,maxjobs=maxJobs)
+print 'All jobs finished, bye.'
Property changes on: trunk/gui/py/yade-multi
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/scripts/multi.py
===================================================================
--- trunk/scripts/multi.py 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/scripts/multi.py 2008-10-28 10:30:53 UTC (rev 1562)
@@ -0,0 +1,45 @@
+#
+# see http://yade.wikia.com/wiki/ScriptParametricStudy#scripts.2Fmulti.py for explanations
+#
+## read parameters from table here
+from yade import utils
+utils.readParamsFromTable(gravity=-9.81,density=2400,initialSpeed=20,noTableOk=True)
+print gravity,density,initialSpeed
+
+o=Omega()
+o.initializers=[
+ StandAloneEngine('PhysicalActionContainerInitializer'),
+ MetaEngine('BoundingVolumeMetaEngine',[EngineUnit('InteractingSphere2AABB'),EngineUnit('InteractingFacet2AABB'),EngineUnit('MetaInteractingGeometry2AABB')])
+ ]
+o.engines=[
+ StandAloneEngine('PhysicalActionContainerReseter'),
+ MetaEngine('BoundingVolumeMetaEngine',[
+ EngineUnit('InteractingSphere2AABB'),
+ EngineUnit('InteractingBox2AABB'),
+ EngineUnit('MetaInteractingGeometry2AABB')
+ ]),
+ StandAloneEngine('PersistentSAPCollider'),
+ MetaEngine('InteractionGeometryMetaEngine',[
+ EngineUnit('InteractingSphere2InteractingSphere4SpheresContactGeometry'),
+ EngineUnit('InteractingBox2InteractingSphere4SpheresContactGeometry')
+ ]),
+ MetaEngine('InteractionPhysicsMetaEngine',[EngineUnit('SimpleElasticRelationships')]),
+ StandAloneEngine('ElasticContactLaw'),
+ DeusExMachina('GravityEngine',{'gravity':[0,0,gravity]}), ## here we use the 'gravity' parameter
+ DeusExMachina('NewtonsDampedLaw',{'damping':0.4})
+]
+o.bodies.append(utils.box([0,50,0],extents=[1,50,1],dynamic=False,color=[1,0,0],young=30e9,poisson=.3,density=density)) ## here we use the density parameter
+o.bodies.append(utils.sphere([0,0,10],1,color=[0,1,0],young=30e9,poisson=.3,density=density)) ## here again
+
+o.bodies[1].phys['velocity']=[0,initialSpeed,0] ## assign initial velocity
+
+o.dt=.8*utils.PWaveTimeStep()
+## o.saveTmp('initial')
+
+# run 30000 iterations
+o.run(30000,True)
+
+# write some results to a common file
+# (we rely on the fact that 2 processes will not write results at exactly the same time)
+# 'a' means to open for appending
+file('multi.out','a').write('%s %g %g %g %g\n'%(o.tags['description'],gravity,density,initialSpeed,o.bodies[1].phys.pos[1]))
Added: trunk/scripts/multi.table
===================================================================
--- trunk/scripts/multi.table 2008-10-27 21:41:21 UTC (rev 1561)
+++ trunk/scripts/multi.table 2008-10-28 10:30:53 UTC (rev 1562)
@@ -0,0 +1,9 @@
+description density initialSpeed ## comment here
+reference 2400 10
+hi_v 2400 20
+lo_v 2400 5 # comment there
+hi_rho 5000 10
+hi_rho_v 5000 20
+hi_rh0_lo_v 5000 5
+
+ # comment-only line
