#!/usr/bin/env python
# -------------------------------------------------------------------------------------------------
# Macroprocessor vith PYthon script inclusion
#
# Written by Franck Wajsburt at LIP6 2009 
# 
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# -------------------------------------------------------------------------------------------------
# requires doctutils package to run rst2html 
#
# TODO exectution step by step
# TODO ecrire un manuel
# TODO faire des tests unitaires
# -------------------------------------------------------------------------------------------------
_date = '2009-12-30 16:00'
_version = '1.07'
_authors = 'Franck Wajsburt'
_organization = '(UPMC/LIP6/SOC)'
__doc__ = '''\
===========================================================================
Macroprocessor vith PYthon script inclusion
 - Authors:      '''+_authors+'''
 - Date:         '''+_date+'''
 - Version:      '''+_version+'''
 - Organisation: '''+_organization+'''
==========================================================================='''

import os, sys, pdb
from optparse import OptionParser
from datetime import datetime

# -------------------------------------------------------------------------------------------------
# os and log utilities
# -------------------------------------------------------------------------------------------------

def _mpyhash(s):
   return "%08x" % (hash(s) & 0xffffffff)

def _exit(error):
   if (error <> ''):
      print >> sys.stderr, "\n*** mpy:", error
   print >> sys.stderr, "*** preprocessing aborted\n"
   exit(1)

def _mpyfindfile(fname, mpyidir):
   "try to find fname in any directory (or subdir) of mpyidir list"
   "if fname is 'stdin' then just returns 'stdin' else returns real filename"
   "if fname does not exist then returns ''"
   if fname == 'stdin':
      return fname
   else:
      for path in mpyidir:
         for root, dirs, files  in os.walk(path):
            for f in files:
               if f == fname: # fname found
                  return os.path.join(root,f)
      return ''

def _mpyopen(fname = '', mpyidir = [], realfname = ''):
   "two behaviors:"
   " 1. try to open realfname but if realfname is '' then 2."
   " 2. try to open fname in any directory OR SUBDIR of mpyidir list1"
   "returns tuple (string_with_file_content, real_file_name)"
   "if fname is 'stdin' then open sys.stdin file"
   if realfname == '':
      realfname = _mpyfindfile(fname,mpyidir)
   if realfname == '':
      _exit("unable to find file %s" % fname)
   if realfname == 'stdin':
      return sys.stdin, fname
   else:
      try: mpyin = open(realfname,'r')
      except IOError, e: _exit(e) # but cannot be open
      return mpyin, realfname

def _mpytree(mess):
   print >> mpyglobals['mpytree'], mess
   mpyglobals['mpytree'].flush()

def _mpylog(mess):
   global mpyglobals
   print >> mpyglobals['mpylog'], mess
   mpyglobals['mpylog'].flush()
   if mpyglobals['mpyverbose']:
      print >> sys.stderr, mess

def _dictlog(dic, exclude=[]):
   for key,val in dic.items():
      for exc in exclude:
         if key[0:len(exc)] == exc: 
            break
      else:  
         _mpylog(' :%s: %s' % (key, str(val)))

# -------------------------------------------------------------------------------------------------
# macro expander
# -------------------------------------------------------------------------------------------------

def _mpy( mpyins  = []    # ['infile1',  ...]  
        , mpytxt  = ''    # 'python code' 
        , mpyout  = ''    # 'outfile' 
        , mpydict = {} ): # global dictionary 

   # to allow _mpy call in inclusion
   exec 'from mpy import mpyglobals, mpygen, mpyexp' in mpydict

   # get arguments from the global dictionary
   global mpyglobals
   mpylog     = mpyglobals['mpylog']     # log file
   mpydebug   = mpyglobals['mpydebug']   # True|False
   mpyverbose = mpyglobals['mpyverbose'] # True|False
   mpywdir    = mpyglobals['mpywdir']    # working directory
   mpyidir    = mpyglobals['mpyidir']    # list of input directory
   mpyoutlist = mpyglobals['mpyoutlist'] # list of opened output
   mpytabwidth= mpyglobals['mpytabwidth']# number of char of a tabulation character
   
   # this is necessary in order to write in mpyout file with just print
   if mpyout not in ('', 'stdout'):
      stdout = sys.stdout
      try: 
         sys.stdout = open( mpywdir+'/'+mpyout, "w")
         mpyoutlist.append(mpywdir+'/'+mpyout)
      except IOError, e: _exit(e)

   opened_mpyin = [] # list of opened mpyin (include)

   # default values, a way to define the variable type
   mpyin = sys.stdin # mpyin file
   mpyfname = ''     # real filename of mpyin file
   mpylineno = 1     # line number in mpyin file
   mpycharno = 0     # character number in mpyin file
   mpylength = 0     # number of character of mpyin file
   level = 0;        # level > 0 in python inclusion
   mode = ''         # '' outside python section, 'eval' | 'code' inside
   buf = ''          # buffer for python section

   # the first item in mpyins is a buffer to exec
   mpyins = [mpytxt] + mpyins

   # for all files in mpyins list (first name is a buffer) {
   nbloop = 0
   for fname in mpyins:

      if nbloop == 0:
         # the fist item of mpyins is a buffer
         mpyin, mpyfname = None, 'buffer'
         mpybuf = fname
         if mpybuf != '':
            lbuf = len(mpybuf)
            if lbuf > 31: lbuf = 31
            if mpydebug == True:
               _mpylog("\n**Expand to file** %s **from text** `%s...`" 
                      % (mpyoutlist[-1], mpybuf[0:lbuf]))
            if mpydebug == True: 
               _dictlog(mpydict, exclude=['_','mpy'])
      else:
         # try to open fnam then initialize all associated counters
         mpyin, mpyfname = _mpyopen(fname,mpyidir)
         mpybuf = mpyin.read(-1)   # all chars of mpyin file
         if __name__ == '__main__':
            _mpylog('\nLog\n'+'-'*40) 
            
         _mpylog("\n**Generate to file** %s **from file** %s"  
                % (mpyoutlist[-1], mpyfname))
         if nbloop == 1 and mpytxt == '': 
            _dictlog(mpydict, exclude=['_','mpy'])

      mpylineno = 1              # line number in mpyin file
      mpycharno = 0              # character number in mpyin file
      mpylinecharno = 0          # character number in current line
      startlineno = 0            # first ligne of python inclusion
      mpylength = len(mpybuf)    # number of character of mpyin file
      level = 0;                 # level > 0 in python inclusion
      mode = ''                  # '' because outside python section at first
      buf = ''                   # buffer for python section
      instr = 0                  # in string boolean (0 outside, 1 inside)
      erasespace = 0             # nb space to erase for current line
      nbspacetab = 0             # nb space to erase for current expands

      # fname may include files, when a file ends, it is maybe an
      # include file and then we must continue to process.
      # This while exits with a break statement
      # while the hierarchical file continues {
      while True:

         # foreach char in current mpybuf {
         while mpycharno < mpylength:

            # get the current char
            c = mpybuf[mpycharno]
            mpycharno += 1

            # in order to get line number of error
            if c == '\n':
               mpylinecharno = 0
               mpylineno += 1
            else:
               mpylinecharno += 1

            # debug mode merges python section with result file
            if mpydebug == True :
               sys.stdout.write(c)

            # outside python section
            if level == 0:
               startlineno = mpylineno
               if c == '[':
                  level += 1
                  mode = 'eval'
               elif c == '{':
                  level += 1
                  mode = 'code'
               else:
                  if mpydebug == False: # in debug mode c is already written
                     sys.stdout.write(c)

            # inside python section
            else:

               # tabulation of expansion
               if erasespace != 0:
                  if c == ' ' :
                     erasespace -= 1
                     continue
                  if c == '\t' :
                     erasespace -= mpytabwidth
                     continue
                  else:
                     if c == '\n':
                        erasespace = nbspacetab
                        buf += c
                        continue
                     else:
                        _exit ( "%s[%d]: %d spaces expected not %c" 
                              % (mpyfname, mpylineno, nbspacetab, c))
               
               # we are inside a string is it a newline ?
               if instr == 1 and c == '\n':
                  erasespace = nbspacetab

               # is it a string delimitor ?
               if buf[-2:]+c == "'''" :
                  instr = 1 - instr
                  if instr == 1:
                     nbspacetab = mpylinecharno-3

               # count [ and {
               if c == '[' :
                  if mode == 'eval':
                     level += 1
                  buf += c
               elif c == '{':
                  if mode == 'code':
                     level += 1
                  buf += c

               # test for evaluation 
               elif c == ']':
                  if level == 0:
                     if startlineno == mpylineno:
                        _exit ( "%s[%d]: unexpected ']'" % (mpyfname, mpylineno))
                     else:
                        _exit ( "%s[%d-%d]: unexpected ']'" % (mpyfname, startlineno, mpylineno))
                  elif mode == 'code':
                     buf += c
                  else: # mode eval
                     level -= 1
                     if level != 0:
                        buf += c
                     else:
                        # test if buf is a file name, if yes includes file
                        # therefore pushes the current file and opens the new
                        incname = _mpyfindfile(buf,mpyidir)
                        if incname != '':
                           if mpydebug == True:
                              _mpylog("\nInclude  to %s from %s" % (mpyout, incname))

                           # push known parameters of the current opened file
                           opened_mpyin.append((mpyin, mpybuf, mpyfname,
                                                mpylineno, mpycharno, mpylength))

                           # old values required for test of recursivity just below
                           oldmpyfname = mpyfname
                           oldmpylineno = mpylineno

                           # open the include file
                           mpyin, mpyfname = _mpyopen(realfname = incname)
                           mpybuf = mpyin.read(-1)
                           mpylineno = 1
                           mpycharno = 0
                           mpylength = len(mpybuf)
                           level = 0
                           buf = mode = ''

                           # test recursive include
                           for f in opened_mpyin[:-1]:
                              if mpyfname in f:
                                 _exit ("%s[%d]: forbidden recursive inclusion on %s"
                                        % (oldmpyfname, oldmpylineno, mpyfname))

                        # just evaluate buf
                        else:
                           try: print eval(buf, mpydict),
                           except Exception, e:
                              if startlineno == mpylineno:
                                 _exit ( "%s[%d]: %s" % (mpyfname, mpylineno, e))
                              else:
                                 _exit ( "%s[%d-%d]: %s" % (mpyfname, startlineno, mpylineno, e))
                           buf = mode = ''

               # test for execute            
               elif c == '}':
                  
                  if level == 0:
                     if startlineno == mpylineno:
                        _exit ( "%s[%d]: unexpected '}'" % (mpyfname, mpylineno))
                     else:
                        _exit ( "%s[%d-%d]: unexpected '}'" % (mpyfname, startlineno, mpylineno))
                  elif mode == 'eval':
                     buf += c
                  else: # mode code
                     level -= 1
                     if level != 0:
                        buf += c
                     else:
                        try:
                           mpydict
                           # print >> sys.stderr, buf
                           exec(buf, mpydict)
                           # pdb.run(buf, mpydict)
                        except Exception, e:
                           if startlineno == mpylineno:
                              _exit ( "%s[%d]: %s" 
                                     % (mpyfname, mpylineno, e))
                           else:
                              _exit ( "%s[%d-%d]: %s" 
                                    % (mpyfname, startlineno, mpylineno, e))
                        buf = mode = ''
               else:
                  buf += c

         # } end while mpycharno < mpylength:

         if mpyfname not in ['stdin', 'buffer']:
            mpyin.close()

         if level <> 0:
            if mode == 'eval':
               _exit ("%s[%d]: expected ']'" % (mpyfname,mpylineno))
            else:
               _exit ("%s[%d]: expected '}'" % (mpyfname,mpylineno))

         # pops included file if there is one, else breaks while
         try:
            mpyin, mpybuf, mpyfname, mpylineno, mpycharno, mpylength = opened_mpyin.pop()
            level = 0
            buf = mode = ''

         except: break

      # } end while True:

      # because the first fname is actually a buffer
      nbloop += 1

   # } end for fname in mpyins:

   # close and redirect sys.stdout
   if mpyout not in ('', 'stdout'):
      sys.stdout.close()
      sys.stdout = stdout
      mpyoutlist.pop()

# -------------------------------------------------------------------------------------------------

def mpygen(mpyin, param):
   "call the macro processor on file mpyin with its own dictionaty, produce a newfile"
   global mpyglobals

   # check parameters
   if type(mpyin) <> str:
      _exit('first argument of mpygen: %s is not a valid filename' % str(mpyin))
   if type(param) <> dict:
      _exit('second argument of mpygen: %s is not a valid dictionary' % str(param))

   # test input filename syntax
   mpyout = os.path.basename(mpyin).split('.')
   if mpyout[-2] <> 'mpy':
      _exit('filename %s has not a valid form (anyroot.mpy.anyext)' % str(mpyin))

   # generate output filename
   hashnumber = _mpyhash(`param`)
   mpyout_base = '.'.join(mpyout[0:-2])
   mpyout = mpyout_base+'_'+hashnumber+'.'+mpyout[-1]
   mpymodelname = mpyout_base+'_'+hashnumber
   param['mpymodelname'] = mpymodelname

   # call the macro processor
   _mpytree('%s* %s <- %s\n' % ('  '*mpyglobals['mpylevelgen'], mpyout, mpyin))
   mpyglobals['mpylevelgen'] += 1
   _mpy(mpyins = [mpyin], mpydict = param, mpyout = mpyout)
   mpyglobals['mpylevelgen'] -= 1
   _mpytree('')

   # return the computed model name
   return mpymodelname

# -------------------------------------------------------------------------------------------------

def mpyexp(txt, param):
   "call the macro processor on a buffer of chars with its own dictionary"
   if type(txt) <> str:
      _exit('argument of mpyexp: must be a string not %s' % type(txt))
   _mpy(mpytxt = txt, mpydict = param)

# -------------------------------------------------------------------------------------------------
# define globals dictionnary
# -------------------------------------------------------------------------------------------------

_parser = OptionParser( 
            usage = "%prog [options] [infiles...]",
            version = "version "+_version, 
            description = "mpy opens all infiles list (default stdin) to macro-process " 
                          "them and writes the result in OFILE (default stdout). "
                          "Infiles are searched in any dir or subdir of PATHS (default "
                          "MPYIDIR environment variable)")

_parser.set_defaults( verbose = False , debug = False , txt = '', tab = 8 
                    , wdir = '.' , idir = '' , output = 'stdout')

_parser.add_option("-v", "--verbose", action = "store_true", dest = "verbose"
                  , help = "trace processing step")
_parser.add_option("-d", "--debug", action = "store_true", dest = "debug" 
                  , help = "do not remove python inclusions")
_parser.add_option("-t", "--txt", dest = "txt" 
                  , help = "text (chars) to macro process")
_parser.add_option("-b", "--tab", dest = "tab" 
                  , help = "tabulation width [default: %default]")
_parser.add_option("-w", "--wdir", dest = "wdir"
                  , help = "define the working directory [default: %default]")
_parser.add_option("-o", "--output", dest = "ofile", default = 'stdout'
                  , help = "define output filename [default: %default] ")
_parser.add_option("-i", "--idir", dest = "idir" 
                  , help = "list of directories (separated with colon ':') "
                           "where to find input files "
                           "(default %prog use MPYPATH environment variable) "
                           "%prog begins to search in the working directory (-w option) "
                           "then in all the directories or subdirectories of the path")

(_options, _mpyins) = _parser.parse_args()

# if no path dirctories are given in command line, getenv MPYIDIR
if _options.idir == '':
   try: _options.idir = os.environ['MPYPATH']
   except: pass

# gets the input files
if len(_mpyins) == 0: 
   _mpyins = ['stdin']

# open log file
if not os.path.isdir(_options.wdir): 
   _exit('%s is not a working directory' % _options.wdir)
try: 
   _mpymetaname = os.path.join(_options.wdir,"mpy_meta.log")
   _mpytreename = os.path.join(_options.wdir,"mpy_tree.log")
   _mpymetafile = open(_mpymetaname,"a")
   _mpytreefile = open(_mpytreename,"a")
   _logname = os.path.join(_options.wdir,"mpy.log")
   _htmlname = os.path.join(_options.wdir,"mpy.html")
except IOError, e: 
   _exit(e) 

mpyglobals = dict( mpyverbose = _options.verbose # True|False
                 , mpydebug = _options.debug # True|False
                 , mpywdir = _options.wdir # working directory
                 , mpyidir = [_options.wdir]+_options.idir.split(':') # split pathes in a list
                 , mpylevelgen = 1 		# level of generation (tree deep)
                 , mpylog  = _mpymetafile
                 , mpytree = _mpytreefile
                 , mpytabwidth = _options.tab
                 , mpyoutlist = [_options.ofile])

# -------------------------------------------------------------------------------------------------
# main function
# -------------------------------------------------------------------------------------------------

if __name__ == '__main__':
   _mpytree('''
========================================================================
MPY: Macroprocessor with PYthon script inclusion (V'''+_version+''')
========================================================================
Generation date '''+datetime.now().strftime("%A, %d. %B %Y %I:%M%p")+'''
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
''')
   _mpytree( '\nTree Generation\n'+'-'*40+'\n\n')
   _mpytree( '* %s <- %s\n' % (_options.ofile, _mpyins[0]))
   _mpy(mpytxt = _options.txt, mpyout = _options.ofile, mpyins = _mpyins, mpydict = {})
   try:
      os.system('cat '+_mpytreename+' '+_mpymetaname+' > '+_logname)
      os.system('rst2html '+_logname+' '+_htmlname)
      os.remove(_mpytreename)
      os.remove(_mpymetaname)
   except:
      pass