yade-dev team mailing list archive

[noreply@xxxxxxxxxxxxx]

------------------------------------------------------------
revno: 2599
committer: Václav Šmilauer <eu@xxxxxxxx>
branch nick: yade
timestamp: Wed 2010-12-08 16:20:55 +0100
message:
  1. yade-batch can run a single job (without parameter table)
  2. Avoid some exceptions in plot
modified:
  core/main/yade-batch.in
  examples/concrete/periodic.py
  examples/concrete/uniax.py
  py/plot.py
  py/utils.py


--
lp:yade
https://code.launchpad.net/~yade-dev/yade/trunk

Your team Yade developers is subscribed to branch lp:yade.
To unsubscribe from this branch go to https://code.launchpad.net/~yade-dev/yade/trunk/+edit-subscription

=== modified file 'core/main/yade-batch.in'
--- core/main/yade-batch.in	2010-12-06 20:05:12 +0000
+++ core/main/yade-batch.in	2010-12-08 15:20:55 +0000
@@ -123,7 +123,9 @@
 		lastFinished=max([j.finished for j in jobs])
 		# FIXME: do not report sum of runnign time of all jobs, only the real timespan
 		ret='<p><span style="background-color:%s">Finished</span>, idle for %s, running time %s since %s.</p>'%('red' if failed else 'lime',t2hhmmss(time.time()-lastFinished),t2hhmmss(sum([j.finished-j.started for j in jobs if j.started is not None])),time.ctime(t0))
-	ret+='<p>Pid %d</p>'%(os.getpid())
+	ret+='<p>Pid %d'%(os.getpid())
+	if opts.globalLog: ret+=', log <a href="/log">%s</a>'%(opts.globalLog)
+	ret+='</p>'
 	allCores,busyCores=set(range(0,maxJobs)),set().union(*(j.cores for j in jobs if j.status=='RUNNING'))
 	ret+='<p>%d cores available, %d used + %d free.</p>'%(maxJobs,nUsedCores,maxJobs-nUsedCores)
 	# show busy and free cores; gives nonsense if not all jobs have CPU affinity set
@@ -146,6 +148,9 @@
 					self.__class__.favicon=base64.b64decode(yade.remote.b64favicon)
 				self.sendHttp(self.__class__.favicon,contentType='image/vnd.microsoft.icon')
 				return
+			elif self.path=='/log' and opts.globalLog:
+				self.sendTextFile(opts.globalLog,refresh=5)
+				return
 			jobMatch=re.match('/jobs/([0-9]+)/(.*)',self.path)
 			if not jobMatch:
 				self.send_error(404,self.path); return
@@ -172,6 +177,7 @@
 			elif rest=='script':
 				self.sendPygmentizedFile(job.script,linenostep=5)
 			elif rest=='table':
+				if not job.table: return
 				self.sendPygmentizedFile(job.table,hl_lines=[job.lineNo],linenostep=1)
 			else: self.send_error(404,self.path)
 		return
@@ -186,7 +192,7 @@
 	def sendTextFile(self,fileName,**headers):
 		if not os.path.exists(fileName): self.send_error(404); return
 		import codecs
-		f=codecs.open(f,encoding='utf-8')
+		f=codecs.open(fileName,encoding='utf-8')
 		self.sendHttp(f.read(),contentType='text/plain;charset=utf-8;',**headers)
 	def sendFile(self,fileName,contentType,**headers):
 		if not os.path.exists(fileName): self.send_error(404); return
@@ -291,9 +297,9 @@
 numCores=getNumCores()
 maxOmpThreads=numCores if 'openmp' in yade.config.features else 1
 
-parser=optparse.OptionParser(usage='%prog [options] TABLE SIMULATION.py',description='%prog runs yade simulation multiple times with different parameters.\n\nSee https://yade-dem.org/sphinx/user.html#batch-queuing-and-execution-yade-batch for details.')
+parser=optparse.OptionParser(usage='%prog [options] [TABLE] SIMULATION.py',description='%prog runs yade simulation multiple times with different parameters.\n\nSee https://yade-dem.org/sphinx/user.html#batch-queuing-and-execution-yade-batch for details.')
 parser.add_option('-j','--jobs',dest='maxJobs',type='int',help="Maximum number of simultaneous threads to run (default: number of cores, further limited by OMP_NUM_THREADS if set by the environment: %d)"%numCores,metavar='NUM',default=numCores)
-parser.add_option('--job-threads',dest='defaultThreads',type='int',help="Default number of threads for one job; can be overridden by per-job with !THREADS (or !OMP_NUM_THREADS) column. Defaults to allocate all available cores (%d) for each job when built with OpenMP, otherwise to 1."%maxOmpThreads,metavar='NUM',default=maxOmpThreads)
+parser.add_option('--job-threads',dest='defaultThreads',type='int',help="Default number of threads for one job; can be overridden by per-job with !THREADS (or !OMP_NUM_THREADS) column. Defaults to allocate all available cores (%d, which might be further limited with --jobs) for each job when built with OpenMP, otherwise to 1."%maxOmpThreads,metavar='NUM',default=-1)
 parser.add_option('--force-threads',action='store_true',dest='forceThreads',help='Force jobs to not use more cores than the maximum (see -j), even if !THREADS colums specifies more.')
 parser.add_option('--log',dest='logFormat',help='Format of job log files -- must contain a % or @, which will be replaced by line number or by description column respectively (default: SIMULATION.@.log)',metavar='FORMAT')
 parser.add_option('--global-log',dest='globalLog',help='Filename where to redirect output of yade-batch itself (as opposed to --log); if not specified (default), stdout/stderr are used',metavar='FILE')
@@ -328,36 +334,45 @@
 	sys.stderr=open(globalLog,"w")
 	sys.stdout=sys.stderr
 
-if len(args)!=2:
-	#print "Exactly two non-option arguments must be specified -- parameter table and script to be run.\n"
+if len(args) not in (1,2):
 	parser.print_help()
 	sys.exit(1)
-table,simul=args[0:2]
+elif len(args)==2:
+	table,simul=args[0:2]
+else:
+	assert(len(args)==1);
+	table=None; simul=args[0]
+
 if not logFormat: logFormat=(simul[:-3] if simul[-3:]=='.py' else simul)+".@.log"
 if (not '%' in logFormat) and ('@' not in logFormat): raise StandardError("Log string must contain at least one of `%', `@'")
 
 print "Will run `%s' on `%s' with nice value %d, output redirected to `%s', using max %d cores."%(executable,simul,nice,logFormat,maxJobs)
 
-reader=yade.utils.TableParamReader(table)
-params=reader.paramDict()
-availableLines=params.keys()
-
-print "Will use table `%s', with available lines"%(table),', '.join([str(i) for i in availableLines])+'.'
-
-if lineList:
-	useLines=[]
-	def numRange2List(s):
-		ret=[]
-		for l in s.split(','):
-			if "-" in l: ret+=range(*[int(s) for s in l.split('-')]); ret+=[ret[-1]+1]
-			else: ret+=[int(l)]
-		return ret
-	useLines0=numRange2List(lineList)
-	for l in useLines0:
-		if l not in availableLines: logging.warn('Skipping unavailable line %d that was requested from the command line.'%l)
-		else: useLines+=[l]
-else: useLines=availableLines
-print "Will use lines ",', '.join([str(i)+' (%s)'%params[i]['description'] for i in useLines])+'.'
+if table:
+	reader=yade.utils.TableParamReader(table)
+	params=reader.paramDict()
+	availableLines=params.keys()
+
+	print "Will use table `%s', with available lines"%(table),', '.join([str(i) for i in availableLines])+'.'
+
+	if lineList:
+		useLines=[]
+		def numRange2List(s):
+			ret=[]
+			for l in s.split(','):
+				if "-" in l: ret+=range(*[int(s) for s in l.split('-')]); ret+=[ret[-1]+1]
+				else: ret+=[int(l)]
+			return ret
+		useLines0=numRange2List(lineList)
+		for l in useLines0:
+			if l not in availableLines: logging.warn('Skipping unavailable line %d that was requested from the command line.'%l)
+			else: useLines+=[l]
+	else: useLines=availableLines
+	print "Will use lines ",', '.join([str(i)+' (%s)'%params[i]['description'] for i in useLines])+'.'
+else:
+	print "Running stand-alone simulation in batch mode."
+	useLines=[-1]
+	params={-1:{'description':'default'}}
 
 jobs=[]
 executables=set()
@@ -365,7 +380,7 @@
 	logFile=logFormat.replace('%',str(l))
 	logFile=logFile.replace('@',params[l]['description'])
 	envVars=[]
-	nCores=opts.defaultThreads
+	nCores=opts.defaultThreads if opts.defaultThreads>0 else opts.maxJobs
 	jobExecutable=executable
 	jobAffinity=opts.affinity
 	jobDebug=opts.debug
@@ -392,7 +407,9 @@
 	for j in range(0,opts.timing if opts.timing>0 else 1):
 		jobNum=len(jobs)
 		logFileNumbered=logFile+('.%d'%j if opts.timing>0 else '')
-		env='PARAM_TABLE=<a href="jobs/%d/table">%s:%d</a> DISPLAY= %s '%(jobNum,table,l,' '.join(envVars))
+		env='YADE_BATCH='
+		if table: env='<a href="jobs/%d/table">%s:%d</a>'%(jobNum,table,l) # keep YADE_BATCH empty (but still defined) if running a single simulation
+		env+=' DISPLAY= %s '%(' '.join(envVars))
 		cmd='%s%s [threadspec] %s -x <a href="jobs/%d/script">%s</a>'%(jobExecutable,' --debug' if jobDebug else '','--nice=%s'%nice if nice!=None else '',i,simul)
 		log='> <a href="jobs/%d/log">%s</a> 2>&1'%(jobNum,pipes.quote(logFileNumbered))
 		hrefCmd=env+cmd+log

=== modified file 'examples/concrete/periodic.py'
--- examples/concrete/periodic.py	2010-09-27 17:47:59 +0000
+++ examples/concrete/periodic.py	2010-12-08 15:20:55 +0000
@@ -189,6 +189,5 @@
 	yade.plot.addData(t=O.time,i=O.iter,eps=strainer.strain[axis],eps_=strainer.strain[axis],sigma=strainer.stress[axis]+isoPrestress,eps1=strainer.strain[ax1],eps2=strainer.strain[ax2],sig1=strainer.stress[ax1],sig2=strainer.stress[ax2],relResid=updater.avgRelResidual)
 
 initTest()
-# sleep forever if run by yade-multi, exit is called from stopIfDamaged
-if os.environ.has_key('PARAM_TABLE'): time.sleep(1e12)
+utils.waitIfBatch()
 

=== modified file 'examples/concrete/uniax.py'
--- examples/concrete/uniax.py	2010-11-07 08:55:43 +0000
+++ examples/concrete/uniax.py	2010-12-08 15:20:55 +0000
@@ -179,10 +179,6 @@
 		})
 plot.plot()
 O.run()
+initTest()
 utils.waitIfBatch()
-#initTest()
-# sleep forever if run by yade-multi, exit is called from stopIfDamaged
-#if os.environ.has_key('PARAM_TABLE'): time.sleep(1e12)
-
-
 

=== modified file 'py/plot.py'
--- py/plot.py	2010-12-05 17:10:06 +0000
+++ py/plot.py	2010-12-08 15:20:55 +0000
@@ -199,8 +199,8 @@
 			pylab.twinx()
 			for d in plots_p_y2:
 				line,=pylab.plot(data[pStrip],data[d[0]],d[1])
-				scatterPt=([0],[0]) if len(data[pStrip])==0 else (data[pStrip][current],data[d[0]][current])
-				scatter=pylab.scatter(scatterPt[0],scatterPt[1],color=line.get_color())
+				scatterPt=[0,0] if len(data[pStrip])==0 else (data[pStrip][current],data[d[0]][current])
+				scatter=pylab.scatter(scatterPt[0] if not math.isnan(scatterPt[0]) else 0,scatterPt[1] if not math.isnan(scatterPt[1]) else 0,color=line.get_color())
 				currLineRefs.append(LineRef(line,scatter,data[pStrip],data[d[0]]))
 			# legend
 			l=pylab.legend([xlateLabel(_p[0]) for _p in plots_p_y2],loc='upper right')

=== modified file 'py/utils.py'
--- py/utils.py	2010-12-06 19:24:20 +0000
+++ py/utils.py	2010-12-08 15:20:55 +0000
@@ -731,7 +731,7 @@
 def runningInBatch():
 	'Tell whether we are running inside the batch or separately.'
 	import os
-	return os.environ.has_key('PARAM_TABLE')
+	return 'YADE_BATCH' in os.environ
 
 def waitIfBatch():
 	'Block the simulation if running inside a batch. Typically used at the end of script so that it does not finish prematurely in batch mode (the execution would be ended in such a case).'
@@ -778,11 +778,11 @@
 	# dictParams is what eventually ends up in yade.params.table (default+specified values)
 	dictDefaults,dictParams,dictAssign={},{},{}
 	import os, __builtin__,re,math
-	if not tableFileLine and not os.environ.has_key('PARAM_TABLE'):
-		if not noTableOk: raise EnvironmentError("PARAM_TABLE is not defined in the environment")
+	if not tableFileLine and ('YADE_BATCH' not in os.environ or os.environ['YADE_BATCH']==''):
+		if not noTableOk: raise EnvironmentError("YADE_BATCH is not defined in the environment")
 		O.tags['line']='l!'
 	else:
-		if not tableFileLine: tableFileLine=os.environ['PARAM_TABLE']
+		if not tableFileLine: tableFileLine=os.environ['YADE_BATCH']
 		env=tableFileLine.split(':')
 		tableFile,tableLine=env[0],int(env[1])
 		allTab=TableParamReader(tableFile).paramDict()