yade-dev team mailing list archive
-
yade-dev team
-
Mailing list archive
-
Message #05661
[Branch ~yade-dev/yade/trunk] Rev 2431: 1. Rename yade-multi to yade-batch; documentation updated
------------------------------------------------------------
revno: 2431
committer: Chiara Modenese <chia@engs-018373>
branch nick: trunk
timestamp: Fri 2010-09-10 12:33:21 +0100
message:
1. Rename yade-multi to yade-batch; documentation updated
2. Make batch save plots when all jobs are done
3. Add tags d.id and id.d (description and id put together)
renamed:
core/main/yade-multi.in => core/main/yade-batch.in
modified:
core/SConscript
core/Scene.cpp
doc/sphinx/upload
doc/sphinx/user.rst
pkg/dem/DataClass/SpherePack.cpp
py/plot.py
py/remote.py
py/utils.py
core/main/yade-batch.in
--
lp:yade
https://code.launchpad.net/~yade-dev/yade/trunk
Your team Yade developers is subscribed to branch lp:yade.
To unsubscribe from this branch go to https://code.launchpad.net/~yade-dev/yade/trunk/+edit-subscription
=== modified file 'core/SConscript'
--- core/SConscript 2010-08-24 12:54:14 +0000
+++ core/SConscript 2010-09-10 11:33:21 +0000
@@ -3,13 +3,13 @@
pyMain='$PREFIX/bin/yade$SUFFIX'
main=env.ScanReplace('main/main.py.in')
-multi=env.ScanReplace('main/yade-multi.in')
+batch=env.ScanReplace('main/yade-batch.in')
env.AlwaysBuild(main)
-env.AlwaysBuild(multi)
+env.AlwaysBuild(batch)
env.InstallAs(pyMain,main)
-env.InstallAs(pyMain+'-multi',multi)
+env.InstallAs(pyMain+'-batch',batch)
env.AddPostAction(pyMain,Chmod(pyMain,0755))
-env.AddPostAction(pyMain+'-multi',Chmod(pyMain+'-multi',0755))
+env.AddPostAction(pyMain+'-batch',Chmod(pyMain+'-batch',0755))
env.Install('$PREFIX/lib/yade$SUFFIX/py/yade',[
env.SharedLibrary('boot',['main/pyboot.cpp'],SHLIBPREFIX='',LIBS=env['LIBS']+['yade-support','core'])
=== modified file 'core/Scene.cpp'
--- core/Scene.cpp 2010-09-02 09:11:04 +0000
+++ core/Scene.cpp 2010-09-10 11:33:21 +0000
@@ -45,8 +45,10 @@
string gecos(pw->pw_gecos), gecos2; size_t p=gecos.find(","); if(p!=string::npos) boost::algorithm::erase_tail(gecos,gecos.size()-p); for(size_t i=0;i<gecos.size();i++){gecos2.push_back(((unsigned char)gecos[i])<128 ? gecos[i] : '?'); }
tags.push_back(boost::algorithm::replace_all_copy(string("author=")+gecos2+" ("+string(pw->pw_name)+"@"+hostname+")"," ","~"));
tags.push_back(string("isoTime="+boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())));
- tags.push_back(string("id="+boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())+"p"+lexical_cast<string>(getpid())));
- tags.push_back(string("description="));
+ string id=boost::posix_time::to_iso_string(boost::posix_time::second_clock::local_time())+"p"+lexical_cast<string>(getpid());
+ tags.push_back("id="+id);
+ tags.push_back("d.id="+id);
+ tags.push_back("id.d="+id);
}
@@ -80,7 +82,7 @@
// ** 2. ** engines
FOREACH(const shared_ptr<Engine>& e, engines){
e->scene=this;
- if(e->dead || !e->isActivated()) continue;
+ if(e->dead || !e->isActivated()) ;
e->action();
if(TimingInfo_enabled) {TimingInfo::delta now=TimingInfo::getNow(); e->timingInfo.nsec+=now-last; e->timingInfo.nExec+=1; last=now;}
}
=== renamed file 'core/main/yade-multi.in' => 'core/main/yade-batch.in'
--- core/main/yade-multi.in 2010-09-09 09:51:23 +0000
+++ core/main/yade-batch.in 2010-09-10 11:33:21 +0000
@@ -1,9 +1,10 @@
#!${pyExecutable}
#Â encoding: utf-8
+# vim: syntax=python
#
# portions © 2008 Václav Šmilauer <eudoxos@xxxxxxxx>
-import os, sys, thread, time, logging, pipes, socket, xmlrpclib, re
+import os, sys, thread, time, logging, pipes, socket, xmlrpclib, re, shutil
#socket.setdefaulttimeout(10)
@@ -50,9 +51,10 @@
self.ensureXmlrpc()
return self.xmlrpcConn.basicInfo()
def updatePlots(self):
+ global opts
if self.status!='RUNNING': return
self.ensureXmlrpc()
- if time.time()-self.plotsLastUpdate<5: return
+ if time.time()-self.plotsLastUpdate<opts.plotTimeout: return
self.plotsLastUpdate=time.time()
img=self.xmlrpcConn.plot()
if not img:
@@ -235,6 +237,8 @@
parser.add_option('--dry-run',action='store_true',dest='dryRun',help='Do not actually run (useful for getting gnuplot only, for instance)',default=False)
parser.add_option('--http-wait',action='store_true',dest='httpWait',help='Do not quit if still serving overview over http repeatedly',default=False)
parser.add_option('--generate-manpage',help='Generate man page documenting this program and exit',dest='manpage',metavar='FILE')
+parser.add_option('--plot-update',type='int',dest='plotAlwaysUpdateTime',help='Interval (in seconds) at which job plots will be updated even if not requested via HTTP. Non-positive values will make the plots not being updated and saved unless requested via HTTP (see --plot-timeout for controlling maximum age of those). Plots are saved at exit under the same name as the log file, with the .log extension removed. (default: 60 seconds)',metavar='TIME',default=60)
+parser.add_option('--plot-timeout',type='int',dest='plotTimeout',help='Maximum age (in seconds) of plots served over HTTP; they will be updated if they are older. (default: 10 seconds)',metavar='TIME',default=10)
opts,args=parser.parse_args()
logFormat,lineList,maxJobs,nice,executable,gnuplotOut,dryRun,httpWait,globalLog=opts.logFormat,opts.lineList,opts.maxJobs,opts.nice,opts.executable,opts.gnuplotOut,opts.dryRun,opts.httpWait,opts.globalLog
@@ -310,16 +314,28 @@
httpLastServe=0
runHttpStatsServer()
+if opts.plotAlwaysUpdateTime>0:
+ # update plots periodically regardless of whether they are requested via HTTP
+ def updateAllPlots():
+ time.sleep(opts.plotAlwaysUpdateTime)
+ for job in jobs: job.updatePlots()
+ thread.start_new_thread(updateAllPlots,())
# OK, go now
if not dryRun: runJobs(jobs,maxJobs)
print 'All jobs finished, total time ',t2hhmmss(sum([j.finished-j.started for j in jobs if j.started is not None]))
+plots=[]
+for j in jobs:
+ if not os.path.exists(j.plotsFile): continue
+ f=(j.log[:-3] if j.log.endswith('.log') else j.log+'.')+yade.remote.plotImgFormat
+ shutil.copy(j.plotsFile,f)
+ plots.append(f)
+if plots: print 'Plot files:',' '.join(plots)
+
# for easy grepping in logfiles:
-print 'Log files:'
-for job in jobs: print job.log,
-print
+print 'Log files:',' '.join([j.log for j in jobs])
if not gnuplotOut:
print 'Bye.'
@@ -352,9 +368,9 @@
gp.write('plot '+','.join(plots))
print "gnuplot",gnuplotOut
print "Plot written, bye."
-if httpWait and time.time()-httpLastServe<10:
+if httpWait and time.time()-httpLastServe<30:
print "(continue serving http until no longer requested as per --http-wait)"
- while time.time()-httpLastServe<10:
+ while time.time()-httpLastServe<30:
time.sleep(1)
yade.Omega().exitNoBacktrace()
=== modified file 'doc/sphinx/upload'
--- doc/sphinx/upload 2010-06-11 22:22:13 +0000
+++ doc/sphinx/upload 2010-09-10 11:33:21 +0000
@@ -1,2 +1,2 @@
#!/bin/sh
-rsync -rv _build/html/ bbeta:www/NoBackup/yade-sphinx
+rsync -rv _build/html/ gamma:www/NoBackup/yade-sphinx
=== modified file 'doc/sphinx/user.rst'
--- doc/sphinx/user.rst 2010-08-15 17:21:04 +0000
+++ doc/sphinx/user.rst 2010-09-10 11:33:21 +0000
@@ -711,8 +711,10 @@
Unique identifier of this Yade instance (or of the instance which created a loded simulation). It is composed of date, time and process number. Useful if you run simulations in parallel and want to avoid overwriting each other's outputs; embed ``O.tags['id']`` in output filenames (either as directory name, or as part of the file's name itself) to avoid it. This is explained in :ref:`batch-output-separate` in detail.
isoTime
Time when simulation was created (with second resolution).
+d.id, id.d
+ Simulation description and id joind by period (and vice-versa). Description is used in batch jobs; in non-batch jobs, these tags are identical to id.
-You can add your own tags by simply assigning value, with the restriction that the left-handside object must be a string:
+You can add your own tags by simply assigning value, with the restriction that the left-handside object must be a string and must not contain ``=``.
.. ipython::
@@ -1122,7 +1124,7 @@
This functionality is used by the batch system (described below) to be informed about individual simulation progress and estimated times. If you want to access this information yourself, you can study :ysrc:`core/main/yade-multi.in` for details.
-Batch queuing and execution (yade-multi)
+Batch queuing and execution (yade-batch)
========================================
Yade features light-weight system for running one simulation with different parameters; it handles assignment of parameter values to python variables in simulation script, scheduling jobs based on number of available and required cores and more. The whole batch consists of 2 files:
@@ -1137,14 +1139,14 @@
The batch can be run as ::
- yade-multi parameters.table simulation.py
+ yade-batch parameters.table simulation.py
and it will intelligently run one simulation for each parameter table line.
Example
--------
-This example is found in :ysrc:`scripts/multi.table` and :ysrc:`scripts/multi.py`.
+This example is found in :ysrc:`scripts/batch.table` and :ysrc:`scripts/batch.py`.
Suppsoe we want to study influence of parameters *density* and *initialVelocity* on position of a sphere falling on fixed box. We create parameter table like this::
@@ -1183,42 +1185,42 @@
Let us see what happens when running the batch::
- \$ yade-multi multi.table multi.py
- Will run `/usr/local/bin/yade-trunk' on `multi.py' with nice value 10, output redirected to `multi.@.log', 4 jobs at a time.
- Will use table `multi.table', with available lines 2, 3, 4, 5, 6, 7.
+ \$ yade-batch batch.table batch.py
+ Will run `/usr/local/bin/yade-trunk' on `batch.py' with nice value 10, output redirected to `batch.@.log', 4 jobs at a time.
+ Will use table `batch.table', with available lines 2, 3, 4, 5, 6, 7.
Will use lines 2 (reference), 3 (hi_v), 4 (lo_v), 5 (hi_rho), 6 (hi_rho_v), 7 (hi_rh0_lo_v).
Master process pid 7030
These lines inform us about general batch information: `nice <http://en.wikipedia.org/wiki/Nice_%28Unix%29>`_ level, log file names, how many cores will be used (4); table name, and line numbers that contain parameters; finally, which lines will be used; master `PID <http://en.wikipedia.org/wiki/Process_identifier>`_ is useful for killing (stopping) the whole batch with the ``kill`` command. ::
Job summary:
- #0 (reference/4): PARAM_TABLE=multi.table:2 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x multi.py > multi.reference.log 2>&1
- #1 (hi_v/4): PARAM_TABLE=multi.table:3 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x multi.py > multi.hi_v.log 2>&1
- #2 (lo_v/4): PARAM_TABLE=multi.table:4 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x multi.py > multi.lo_v.log 2>&1
- #3 (hi_rho/4): PARAM_TABLE=multi.table:5 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x multi.py > multi.hi_rho.log 2>&1
- #4 (hi_rho_v/4): PARAM_TABLE=multi.table:6 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x multi.py > multi.hi_rho_v.log 2>&1
- #5 (hi_rh0_lo_v/4): PARAM_TABLE=multi.table:7 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x multi.py > multi.hi_rh0_lo_v.log 2>&1
+ #0 (reference/4): PARAM_TABLE=batch.table:2 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x batch.py > batch.reference.log 2>&1
+ #1 (hi_v/4): PARAM_TABLE=batch.table:3 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x batch.py > batch.hi_v.log 2>&1
+ #2 (lo_v/4): PARAM_TABLE=batch.table:4 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x batch.py > batch.lo_v.log 2>&1
+ #3 (hi_rho/4): PARAM_TABLE=batch.table:5 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x batch.py > batch.hi_rho.log 2>&1
+ #4 (hi_rho_v/4): PARAM_TABLE=batch.table:6 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x batch.py > batch.hi_rho_v.log 2>&1
+ #5 (hi_rh0_lo_v/4): PARAM_TABLE=batch.table:7 DISPLAY= /usr/local/bin/yade-trunk --threads=4 --nice=10 -x batch.py > batch.hi_rh0_lo_v.log 2>&1
displays all jobs with command-lines that will be run for each of them. At this moment, the batch starts to be run. ::
#0 (reference/4) started on Tue Apr 13 13:59:32 2010
- #0 (reference/4) done (exit status 0), duration 00:00:01, log multi.reference.log
+ #0 (reference/4) done (exit status 0), duration 00:00:01, log batch.reference.log
#1 (hi_v/4) started on Tue Apr 13 13:59:34 2010
- #1 (hi_v/4) done (exit status 0), duration 00:00:01, log multi.hi_v.log
+ #1 (hi_v/4) done (exit status 0), duration 00:00:01, log batch.hi_v.log
#2 (lo_v/4) started on Tue Apr 13 13:59:35 2010
- #2 (lo_v/4) done (exit status 0), duration 00:00:01, log multi.lo_v.log
+ #2 (lo_v/4) done (exit status 0), duration 00:00:01, log batch.lo_v.log
#3 (hi_rho/4) started on Tue Apr 13 13:59:37 2010
- #3 (hi_rho/4) done (exit status 0), duration 00:00:01, log multi.hi_rho.log
+ #3 (hi_rho/4) done (exit status 0), duration 00:00:01, log batch.hi_rho.log
#4 (hi_rho_v/4) started on Tue Apr 13 13:59:38 2010
- #4 (hi_rho_v/4) done (exit status 0), duration 00:00:01, log multi.hi_rho_v.log
+ #4 (hi_rho_v/4) done (exit status 0), duration 00:00:01, log batch.hi_rho_v.log
#5 (hi_rh0_lo_v/4) started on Tue Apr 13 13:59:40 2010
- #5 (hi_rh0_lo_v/4) done (exit status 0), duration 00:00:01, log multi.hi_rh0_lo_v.log
+ #5 (hi_rh0_lo_v/4) done (exit status 0), duration 00:00:01, log batch.hi_rh0_lo_v.log
information about job status changes is being printed, until::
All jobs finished, total time 00:00:08
Log files:
- multi.reference.log multi.hi_v.log multi.lo_v.log multi.hi_rho.log multi.hi_rho_v.log multi.hi_rh0_lo_v.log
+ batch.reference.log batch.hi_v.log batch.lo_v.log batch.hi_rho.log batch.hi_rho_v.log batch.hi_rh0_lo_v.log
Bye.
.. batch-output-separate:
@@ -1260,9 +1262,9 @@
Merging gnuplot from individual jobs
-------------------------------------
-Frequently, it is desirable to obtain single figure for all jobs in the batch, for comparison purposes. Somewhat heiristic way for this functionality is provided by the batch system. ``yade-multi`` must be run with the ``--gnuplot`` option, specifying some file name that will be used for the merged figure::
+Frequently, it is desirable to obtain single figure for all jobs in the batch, for comparison purposes. Somewhat heiristic way for this functionality is provided by the batch system. ``yade-batch`` must be run with the ``--gnuplot`` option, specifying some file name that will be used for the merged figure::
- yade-trunk --gnuplot merged.gnuplot multi.table multi.py
+ yade-trunk --gnuplot merged.gnuplot batch.table batch.py
Data are collected in usual way during the simulation (using :yref:`yade.plot.addData`) and saved to gnuplot file via :yref:`yade.plot.saveGnuplot` (it creates 2 files: gnuplot command file and compressed data file). The batch system *scans*, once the job is finished, log file for line of the form ``gnuplot [something]``. Therefore, in onrder to print this *magic line* we put::
=== modified file 'pkg/dem/DataClass/SpherePack.cpp'
--- pkg/dem/DataClass/SpherePack.cpp 2010-09-09 14:02:49 +0000
+++ pkg/dem/DataClass/SpherePack.cpp 2010-09-10 11:33:21 +0000
@@ -33,7 +33,7 @@
python::extract<python::tuple> tup(t[0]);
if(tup.check()) { pack.push_back(Sph(Vector3r(python::extract<double>(tup[0]),python::extract<double>(tup[1]),python::extract<double>(tup[2]))),python::extract<double>(t[1])); continue; }
#endif
- PyErr_SetString(PyExc_TypeError, "List elements must be (tuple or Vector3, float)!");
+ PyErr_SetString(PyExc_TypeError, "List elements must be (Vector3, float)!");
python::throw_error_already_set();
}
};
=== modified file 'py/plot.py'
--- py/plot.py 2010-09-09 14:02:49 +0000
+++ py/plot.py 2010-09-10 11:33:21 +0000
@@ -139,7 +139,7 @@
if subPlots:
# compute number of rows and colums for plots we have
subCols=int(round(math.sqrt(len(plots)))); subRows=int(math.ceil(len(plots)*1./subCols))
- print 'subplot',subCols,subRows
+ #print 'subplot',subCols,subRows
for nPlot,p in enumerate(plots.keys()):
pStrip=p.strip()
if not subPlots: pylab.figure()
=== modified file 'py/remote.py'
--- py/remote.py 2010-09-09 09:51:23 +0000
+++ py/remote.py 2010-09-10 11:33:21 +0000
@@ -7,7 +7,7 @@
"""
import SocketServer,xmlrpclib,socket
-import sys,time,os
+import sys,time,os,math
from yade import *
import yade.runtime
@@ -30,7 +30,9 @@
if len(plot.plots)==0: return None
fig=plot.plot(subPlots=True,noShow=True)
img=O.tmpFilename()+'.'+plotImgFormat
- fig.savefig(img,dpi=200)
+ sqrtFigs=math.sqrt(len(plot.plots))
+ fig.set_size_inches(5*sqrtFigs,7*sqrtFigs)
+ fig.savefig(img)
f=open(img,'rb'); data=f.read(); f.close(); os.remove(img)
#print 'returning '+plotImgFormat
return xmlrpclib.Binary(data)
=== modified file 'py/utils.py'
--- py/utils.py 2010-09-10 11:30:21 +0000
+++ py/utils.py 2010-09-10 11:33:21 +0000
@@ -750,9 +750,11 @@
Assigned tags:
- * *description* column is assigned to Omega().tags['description']; this column is synthesized if absent (see :yref:`yade.utils.TableParamReader`)
- * Omega().tags['params']="name1=val1,name2=val2,…"
- * Omega().tags['defaultParams']="unassignedName1=defaultValue1,…"
+ * *description* column is assigned to Omega().tags['description']; this column is synthesized if absent (see :yref:`yade.utils.TableParamReader`);
+ * ``Omega().tags['params']="name1=val1,name2=val2,…"``
+ * ``Omega().tags['defaultParams']="unassignedName1=defaultValue1,…"``
+ * ``Omega().tags['d.id']=O.tags['id']+'.'+O.tags['description']``
+ * ``Omega().tags['id.d']=O.tags['description']+'.'+O.tags['id']``
All parameters (default as well as settable) are saved using :yref:`yade.utils.saveVars`\ ``('table')``.
@@ -786,6 +788,7 @@
vv=allTab[tableLine]
O.tags['line']='l%d'%tableLine
O.tags['description']=vv['description']
+ O.tags['d.id']=O.tags['id']+'.'+O.tags['description']; O.tags['id.d']=O.tags['description']+'.'+O.tags['id']
# assign values specified in the table to python vars
# !something cols are skipped, those are env vars we don't treat at all (they are contained in description, though)
for col in vv.keys():
