Context Navigation

source: src/mpy.py @ 4

Last change on this file since 4 was 4, checked in by franck, 14 years ago
add TODO list :-)
Property svn:executable set to ``*
File size: 20.8 KB

Line
1	#!/usr/bin/env python
2	# -------------------------------------------------------------------------------------------------
3	# Macroprocessor vith PYthon script inclusion
4	#
5	# Written by Franck Wajsburt at LIP6 2009
6	#
7	# This program is free software; you can redistribute it and/or
8	# modify it under the terms of the GNU General Public License as
9	# published by the Free Software Foundation; either version 2 of the
10	# License, or (at your option) any later version.
11	#
12	# This program is distributed in the hope that it will be useful, but
13	# WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15	# General Public License for more details.
16	#
17	# You should have received a copy of the GNU General Public License
18	# along with this program; if not, write to the Free Software
19	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20	# -------------------------------------------------------------------------------------------------
21	# requires doctutils package to run rst2html
22	#
23	# TODO step by step execution
24	# TODO write a manual
25	# TODO add unitary tests
26	# TODO add [ and { despecialization
27	# TODO save the directory exploration to do it only once per session
28	# -------------------------------------------------------------------------------------------------
29	_date = '2009-12-30 16:00'
30	_version = '1.07'
31	_authors = 'Franck Wajsburt'
32	_organization = '(UPMC/LIP6/SOC)'
33	__doc__ = '''\
34	===========================================================================
35	Macroprocessor vith PYthon script inclusion
36	- Authors: '''+_authors+'''
37	- Date: '''+_date+'''
38	- Version: '''+_version+'''
39	- Organisation: '''+_organization+'''
40	==========================================================================='''
41
42	import os, sys, pdb
43	from optparse import OptionParser
44	from datetime import datetime
45
46	# -------------------------------------------------------------------------------------------------
47	# os and log utilities
48	# -------------------------------------------------------------------------------------------------
49
50	def _mpyhash(s):
51	return "%08x" % (hash(s) & 0xffffffff)
52
53	def _exit(error):
54	if (error <> ''):
55	print >> sys.stderr, "\n*** mpy:", error
56	print >> sys.stderr, "*** preprocessing aborted\n"
57	exit(1)
58
59	def _mpyfindfile(fname, mpyidir):
60	"try to find fname in any directory (or subdir) of mpyidir list"
61	"if fname is 'stdin' then just returns 'stdin' else returns real filename"
62	"if fname does not exist then returns ''"
63	if fname == 'stdin':
64	return fname
65	else:
66	for path in mpyidir:
67	for root, dirs, files in os.walk(path):
68	for f in files:
69	if f == fname: # fname found
70	return os.path.join(root,f)
71	return ''
72
73	def _mpyopen(fname = '', mpyidir = [], realfname = ''):
74	"two behaviors:"
75	" 1. try to open realfname but if realfname is '' then 2."
76	" 2. try to open fname in any directory OR SUBDIR of mpyidir list1"
77	"returns tuple (string_with_file_content, real_file_name)"
78	"if fname is 'stdin' then open sys.stdin file"
79	if realfname == '':
80	realfname = _mpyfindfile(fname,mpyidir)
81	if realfname == '':
82	_exit("unable to find file %s" % fname)
83	if realfname == 'stdin':
84	return sys.stdin, fname
85	else:
86	try: mpyin = open(realfname,'r')
87	except IOError, e: _exit(e) # but cannot be open
88	return mpyin, realfname
89
90	def _mpytree(mess):
91	print >> mpyglobals['mpytree'], mess
92	mpyglobals['mpytree'].flush()
93
94	def _mpylog(mess):
95	global mpyglobals
96	print >> mpyglobals['mpylog'], mess
97	mpyglobals['mpylog'].flush()
98	if mpyglobals['mpyverbose']:
99	print >> sys.stderr, mess
100
101	def _dictlog(dic, exclude=[]):
102	for key,val in dic.items():
103	for exc in exclude:
104	if key[0:len(exc)] == exc:
105	break
106	else:
107	_mpylog(' :%s: %s' % (key, str(val)))
108
109	# -------------------------------------------------------------------------------------------------
110	# macro expander
111	# -------------------------------------------------------------------------------------------------
112
113	def _mpy( mpyins = [] # ['infile1', ...]
114	, mpytxt = '' # 'python code'
115	, mpyout = '' # 'outfile'
116	, mpydict = {} ): # global dictionary
117
118	# to allow _mpy call in inclusion
119	exec 'from mpy import mpyglobals, mpygen, mpyexp' in mpydict
120
121	# get arguments from the global dictionary
122	global mpyglobals
123	mpylog = mpyglobals['mpylog'] # log file
124	mpylist = mpyglobals['mpylist'] # file list log file
125	mpydebug = mpyglobals['mpydebug'] # True\|False
126	mpyverbose = mpyglobals['mpyverbose'] # True\|False
127	mpywdir = mpyglobals['mpywdir'] # working directory
128	mpyidir = mpyglobals['mpyidir'] # list of input directory
129	mpyoutlist = mpyglobals['mpyoutlist'] # list of opened output
130	mpytabwidth= mpyglobals['mpytabwidth']# number of char of a tabulation character
131
132	# this is necessary in order to write in mpyout file with just print
133	if mpyout not in ('', 'stdout'):
134	stdout = sys.stdout
135	try:
136	sys.stdout = open( mpywdir+'/'+mpyout, "w")
137	mpyoutlist.append(mpywdir+'/'+mpyout)
138	mpylist.write(mpywdir+'/'+mpyout+'\n')
139	except IOError, e: _exit(e)
140
141	opened_mpyin = [] # list of opened mpyin (include)
142
143	# default values, a way to define the variable type
144	mpyin = sys.stdin # mpyin file
145	mpyfname = '' # real filename of mpyin file
146	mpylineno = 1 # line number in mpyin file
147	mpycharno = 0 # character number in mpyin file
148	mpylength = 0 # number of character of mpyin file
149	level = 0; # level > 0 in python inclusion
150	mode = '' # '' outside python section, 'eval' \| 'code' inside
151	buf = '' # buffer for python section
152
153	# the first item in mpyins is a buffer to exec
154	mpyins = [mpytxt] + mpyins
155
156	# for all files in mpyins list (first name is a buffer) {
157	nbloop = 0
158	for fname in mpyins:
159
160	if nbloop == 0:
161	# the fist item of mpyins is a buffer
162	mpyin, mpyfname = None, 'buffer'
163	mpybuf = fname
164	if mpybuf != '':
165	lbuf = len(mpybuf)
166	if lbuf > 31: lbuf = 31
167	if mpydebug == True:
168	_mpylog("\nExpand to file %s from text `%s...`"
169	% (mpyoutlist[-1], mpybuf[0:lbuf]))
170	if mpydebug == True:
171	_dictlog(mpydict, exclude=['_','mpy'])
172	else:
173	# try to open fnam then initialize all associated counters
174	mpyin, mpyfname = _mpyopen(fname,mpyidir)
175	mpybuf = mpyin.read(-1) # all chars of mpyin file
176	if __name__ == '__main__':
177	_mpylog('\nLog\n'+'-'*40)
178
179	_mpylog("\nGenerate to file %s from file %s"
180	% (mpyoutlist[-1], mpyfname))
181	if nbloop == 1 and mpytxt == '':
182	_dictlog(mpydict, exclude=['_','mpy'])
183
184	mpylineno = 1 # line number in mpyin file
185	mpycharno = 0 # character number in mpyin file
186	mpylinecharno = 0 # character number in current line
187	startlineno = 0 # first ligne of python inclusion
188	mpylength = len(mpybuf) # number of character of mpyin file
189	level = 0; # level > 0 in python inclusion
190	mode = '' # '' because outside python section at first
191	buf = '' # buffer for python section
192	instr = 0 # in string boolean (0 outside, 1 inside)
193	erasespace = 0 # nb space to erase for current line
194	nbspacetab = 0 # nb space to erase for current expands
195
196	# fname may include files, when a file ends, it is maybe an
197	# include file and then we must continue to process.
198	# This while exits with a break statement
199	# while the hierarchical file continues {
200	while True:
201
202	# foreach char in current mpybuf {
203	while mpycharno < mpylength:
204
205	# get the current char
206	c = mpybuf[mpycharno]
207	mpycharno += 1
208
209	# in order to get line number of error
210	if c == '\n':
211	mpylinecharno = 0
212	mpylineno += 1
213	else:
214	mpylinecharno += 1
215
216	# debug mode merges python section with result file
217	if mpydebug == True :
218	sys.stdout.write(c)
219
220	# outside python section
221	if level == 0:
222	startlineno = mpylineno
223	if c == '[':
224	level += 1
225	mode = 'eval'
226	elif c == '{':
227	level += 1
228	mode = 'code'
229	else:
230	if mpydebug == False: # in debug mode c is already written
231	sys.stdout.write(c)
232
233	# inside python section
234	else:
235
236	# tabulation of expansion
237	if erasespace != 0:
238	if c == ' ' :
239	erasespace -= 1
240	continue
241	if c == '\t' :
242	erasespace -= mpytabwidth
243	continue
244	else:
245	if c == '\n':
246	erasespace = nbspacetab
247	buf += c
248	continue
249	else:
250	_exit ( "%s[%d]: %d spaces expected not %c"
251	% (mpyfname, mpylineno, nbspacetab, c))
252
253	# we are inside a string is it a newline ?
254	if instr == 1 and c == '\n':
255	erasespace = nbspacetab
256
257	# is it a string delimitor ?
258	if buf[-2:]+c == "'''" :
259	instr = 1 - instr
260	if instr == 1:
261	nbspacetab = mpylinecharno-3
262
263	# count [ and {
264	if c == '[' :
265	if mode == 'eval':
266	level += 1
267	buf += c
268	elif c == '{':
269	if mode == 'code':
270	level += 1
271	buf += c
272
273	# test for evaluation
274	elif c == ']':
275	if level == 0:
276	if startlineno == mpylineno:
277	_exit ( "%s[%d]: unexpected ']'" % (mpyfname, mpylineno))
278	else:
279	_exit ( "%s[%d-%d]: unexpected ']'" % (mpyfname, startlineno, mpylineno))
280	elif mode == 'code':
281	buf += c
282	else: # mode eval
283	level -= 1
284	if level != 0:
285	buf += c
286	else:
287	# test if buf is a file name, if yes includes file
288	# therefore pushes the current file and opens the new
289	incname = _mpyfindfile(buf,mpyidir)
290	if incname != '':
291	if mpydebug == True:
292	_mpylog("\nInclude to %s from %s" % (mpyout, incname))
293
294	# push known parameters of the current opened file
295	opened_mpyin.append((mpyin, mpybuf, mpyfname,
296	mpylineno, mpycharno, mpylength))
297
298	# old values required for test of recursivity just below
299	oldmpyfname = mpyfname
300	oldmpylineno = mpylineno
301
302	# open the include file
303	mpyin, mpyfname = _mpyopen(realfname = incname)
304	mpybuf = mpyin.read(-1)
305	mpylineno = 1
306	mpycharno = 0
307	mpylength = len(mpybuf)
308	level = 0
309	buf = mode = ''
310
311	# test recursive include
312	for f in opened_mpyin[:-1]:
313	if mpyfname in f:
314	_exit ("%s[%d]: forbidden recursive inclusion on %s"
315	% (oldmpyfname, oldmpylineno, mpyfname))
316
317	# just evaluate buf
318	else:
319	try: print eval(buf, mpydict),
320	except Exception, e:
321	if startlineno == mpylineno:
322	_exit ( "%s[%d]: %s" % (mpyfname, mpylineno, e))
323	else:
324	_exit ( "%s[%d-%d]: %s" % (mpyfname, startlineno, mpylineno, e))
325	buf = mode = ''
326
327	# test for execute
328	elif c == '}':
329
330	if level == 0:
331	if startlineno == mpylineno:
332	_exit ( "%s[%d]: unexpected '}'" % (mpyfname, mpylineno))
333	else:
334	_exit ( "%s[%d-%d]: unexpected '}'" % (mpyfname, startlineno, mpylineno))
335	elif mode == 'eval':
336	buf += c
337	else: # mode code
338	level -= 1
339	if level != 0:
340	buf += c
341	else:
342	try:
343	mpydict
344	# print >> sys.stderr, buf
345	exec(buf, mpydict)
346	# pdb.run(buf, mpydict)
347	except Exception, e:
348	if startlineno == mpylineno:
349	_exit ( "%s[%d]: %s"
350	% (mpyfname, mpylineno, e))
351	else:
352	_exit ( "%s[%d-%d]: %s"
353	% (mpyfname, startlineno, mpylineno, e))
354	buf = mode = ''
355	else:
356	buf += c
357
358	# } end while mpycharno < mpylength:
359
360	if mpyfname not in ['stdin', 'buffer']:
361	mpyin.close()
362
363	if level <> 0:
364	if mode == 'eval':
365	_exit ("%s[%d]: expected ']'" % (mpyfname,mpylineno))
366	else:
367	_exit ("%s[%d]: expected '}'" % (mpyfname,mpylineno))
368
369	# pops included file if there is one, else breaks while
370	try:
371	mpyin, mpybuf, mpyfname, mpylineno, mpycharno, mpylength = opened_mpyin.pop()
372	level = 0
373	buf = mode = ''
374
375	except: break
376
377	# } end while True:
378
379	# because the first fname is actually a buffer
380	nbloop += 1
381
382	# } end for fname in mpyins:
383
384	# close and redirect sys.stdout
385	if mpyout not in ('', 'stdout'):
386	sys.stdout.close()
387	sys.stdout = stdout
388	mpyoutlist.pop()
389
390	# -------------------------------------------------------------------------------------------------
391
392	def mpygen(mpyin, param):
393	"call the macro processor on file mpyin with its own dictionaty, produce a newfile"
394	global mpyglobals
395
396	# check parameters
397	if type(mpyin) <> str:
398	_exit('first argument of mpygen: %s is not a valid filename' % str(mpyin))
399	if type(param) <> dict:
400	_exit('second argument of mpygen: %s is not a valid dictionary' % str(param))
401
402	# test input filename syntax
403	mpyout = os.path.basename(mpyin).split('.')
404	if mpyout[-2] <> 'mpy':
405	_exit('filename %s has not a valid form (anyroot.mpy.anyext)' % str(mpyin))
406
407	# generate output filename
408	hashnumber = _mpyhash(`param`)
409	mpyout_base = '.'.join(mpyout[0:-2])
410	mpyout = mpyout_base+'_'+hashnumber+'.'+mpyout[-1]
411	mpymodelname = mpyout_base+'_'+hashnumber
412	param['mpymodelname'] = mpymodelname
413
414	# call the macro processor
415	_mpytree('%s* %s <- %s\n' % (' '*mpyglobals['mpylevelgen'], mpyout, mpyin))
416	mpyglobals['mpylevelgen'] += 1
417	_mpy(mpyins = [mpyin], mpydict = param, mpyout = mpyout)
418	mpyglobals['mpylevelgen'] -= 1
419	_mpytree('')
420
421	# return the computed model name
422	return mpymodelname
423
424	# -------------------------------------------------------------------------------------------------
425
426	def mpyexp(txt, param):
427	"call the macro processor on a buffer of chars with its own dictionary"
428	if type(txt) <> str:
429	_exit('argument of mpyexp: must be a string not %s' % type(txt))
430	_mpy(mpytxt = txt, mpydict = param)
431
432	# -------------------------------------------------------------------------------------------------
433	# define globals dictionnary
434	# -------------------------------------------------------------------------------------------------
435
436	_parser = OptionParser(
437	usage = "%prog [options] [infiles...]",
438	version = "version "+_version,
439	description = "mpy opens all infiles list (default stdin) to macro-process "
440	"them and writes the result in OFILE (default stdout). "
441	"Infiles are searched in any dir or subdir of PATHS (default "
442	"MPYIDIR environment variable)")
443
444	_parser.set_defaults( verbose = False , debug = False , txt = '', tab = 8
445	, wdir = '.' , idir = '' , output = 'stdout')
446
447	_parser.add_option("-v", "--verbose", action = "store_true", dest = "verbose"
448	, help = "trace processing step")
449	_parser.add_option("-d", "--debug", action = "store_true", dest = "debug"
450	, help = "do not remove python inclusions")
451	_parser.add_option("-t", "--txt", dest = "txt"
452	, help = "text (chars) to macro process")
453	_parser.add_option("-b", "--tab", dest = "tab"
454	, help = "tabulation width [default: %default]")
455	_parser.add_option("-w", "--wdir", dest = "wdir"
456	, help = "define the working directory [default: %default]")
457	_parser.add_option("-o", "--output", dest = "ofile", default = 'stdout'
458	, help = "define output filename [default: %default] ")
459	_parser.add_option("-n", "--name", dest = "name", default = 'mpy'
460	, help = "define instance name, basename for logs [default: 'mpy']")
461	_parser.add_option("-i", "--idir", dest = "idir"
462	, help = "list of directories (separated with colon ':') "
463	"where to find input files "
464	"(default %prog use MPYPATH environment variable) "
465	"%prog begins to search in the working directory (-w option) "
466	"then in all the directories or subdirectories of the path")
467
468	(_options, _mpyins) = _parser.parse_args()
469
470	# if no path dirctories are given in command line, getenv MPYIDIR
471	if _options.idir == '':
472	try: _options.idir = os.environ['MPYPATH']
473	except: pass
474
475	# gets the input files
476	if len(_mpyins) == 0:
477	_mpyins = ['stdin']
478
479	# open log file
480	if not os.path.isdir(_options.wdir):
481	_exit('%s is not a working directory' % _options.wdir)
482	try:
483	_mpymetaname = os.path.join(_options.wdir,_options.name+"_meta.log")
484	_mpytreename = os.path.join(_options.wdir,_options.name+"_tree.log")
485	_mpylistname = os.path.join(_options.wdir,_options.name+".list")
486	_mpymetafile = open(_mpymetaname,"a")
487	_mpytreefile = open(_mpytreename,"a")
488	_mpylistfile = open(_mpylistname,"a")
489	_logname = os.path.join(_options.wdir, _options.name+".log")
490	_htmlname = os.path.join(_options.wdir,_options.name+".html")
491	except IOError, e:
492	_exit(e)
493
494	mpyglobals = dict( mpyverbose = _options.verbose # True\|False
495	, mpydebug = _options.debug # True\|False
496	, mpywdir = _options.wdir # working directory
497	, mpyidir = [_options.wdir]+_options.idir.split(':') # split pathes in a list
498	, mpylevelgen = 1 # level of generation (tree deep)
499	, mpylog = _mpymetafile
500	, mpytree = _mpytreefile
501	, mpylist = _mpylistfile
502	, mpytabwidth = _options.tab
503	, mpyoutlist = [_options.ofile])
504
505	# -------------------------------------------------------------------------------------------------
506	# main function
507	# -------------------------------------------------------------------------------------------------
508
509	if __name__ == '__main__':
510	_mpytree('''
511	========================================================================
512	MPY: Macroprocessor with PYthon script inclusion (V'''+_version+''')
513	========================================================================
514	Generation date '''+datetime.now().strftime("%A, %d. %B %Y %I:%M%p")+'''
515	~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
516	''')
517	_mpytree( '\nTree Generation\n'+'-'*40+'\n\n')
518	_mpytree( '* %s <- %s\n' % (_options.ofile, _mpyins[0]))
519	_mpy(mpytxt = _options.txt, mpyout = _options.ofile, mpyins = _mpyins, mpydict = {})
520	try:
521	os.system('cat '+_mpytreename+' '+_mpymetaname+' > '+_logname)
522	os.remove(_mpytreename)
523	os.remove(_mpymetaname)
524	os.system('rst2html '+_logname+' '+_htmlname+' 2> /dev/null')
525	except:
526	pass

Note: See TracBrowser for help on using the repository browser.

Download in other formats: