#Copyright ReportLab Europe Ltd. 2000-2004
#history www.reportlab.co.uk/rl-cgi/viewcvs.cgi/rlextra/utils/appmonitor.py

"""ReportLab Application Monitor.

This provides utilities to help manage and verify software installations.
The basic problems are that source and binary distriubutions can get
messed up at many points in between our CVS repository and a customer's
live site.  So we need a system to verify integrity.  This should be
easy for customers' configuration management staff to understand.

We have a class which sits over a directory and knows how to do
various things to the files under it.  This can be subclassed
whenever the desired behaviour is not quite right.  The base
class should be good enough to do the right things for standard
ReportLab apps with python and preppy files.  Main functions
are
- help put together a manifest file in development
- generate a signed, checkable 'source manifest' when we
  ship source
- verify the above in a custromer's source environment
  and/or warn of changes
- compile the app, generating pyc files etc.
- generate a signed and checkable 'compiled manifest'
  at the moment of compilation
- if running in live, verify the signatures of the
  compiled files
- warn of stuff which perhaps should not be there,
  e.g. source files in a compiled environment, extra
  modules which should be deprecated etc.

An 'application' will have a manifest we maintain in version
control which lists the files that matter.  By convention
this is filelist.txt.

When we ship something, we can include a manifest_source.txt
with checksums.  This is NOT version controlled - it cannnot
be as checking it in will change its checksum :-)  It can
be generated by a script which makes the zip/tgz for
the customer.

When they run the compile action, it generates a manifest_compiled.txt
with checksums for all the pycs and binary files.  This is promoted
live with the rest of the code.
"""
__version__=''' $Id: appmonitor.py 13580 2004-07-19 16:41:50Z rgbecker $ '''
import sys, os, time, string

from rlextra.preppy import preppy
import pprint
import py_compile
import stat
import shutil
import zipfile
import gloop
from manifester import Manifester
import fnmatch

def getCVSFilesInDirectory(directory):
    """Helper for a method defined below.  Returns a list of the files
    which are listed in the CVS entries.  Barfs if not all are present."""
    if not os.path.isdir(directory + os.sep + 'CVS'):
        return []

    cvsEntries = open(directory + os.sep + 'CVS' + os.sep + 'Entries').readlines()
    filenames = []
    for entry in cvsEntries:
        if len(entry) > 0:
            words = string.split(entry, '/')
            if len(words) > 1:
                filename = words[1]
                fullName = directory + os.sep + filename
                assert os.path.exists(fullName), 'file "%s" listed in CVS but not present!' % fullName
                if os.path.isfile(fullName):
                    filenames.append(fullName)
    filenames.sort()
    return filenames

class AppMonitor(Manifester):
    flName  = 'filelist.txt'                        #name of the original packing list
    ssmName = 'source_manifest.txt'                 #name of the signed source manifest
    stmName = 'target_manifest.txt'                 #name of the signed target manifest
    _force_ssmName = 1                              #force ssmName into target list
    # these may have line ending variations
    def __init__(self, dirname=None, id='', verbose=0, fileList=None):
        Manifester.__init__(self,dirname=dirname,id=id,verbose=verbose)

        self._sourceFiles = fileList
        # when checks fail, track the list of filenames here
        # but continue to a full report.
        self.source_failed = []
        self.source_missing = []
        self.target_failed = []
        self.target_missing = []
        self.unexpected = []

        #can optionally supply a list of filenames, instead of
        #reading a list file name at top level.
        self.suppliedFileList = fileList

    def getList(self):
        "Returns list of files"
        if not self._sourceFiles:
            #read filelist.txt at top level
            lines = open(self.flName,"r").readlines()
            self._sourceFiles = self.stripComments(lines)
        return self._sourceFiles

    def writeSignedSourceManifest(self):
        "Generates signed source manifest"
        self.writeSignedManifest(self.getList(),self.ssmName,_id=1)

    def verifySignedSourceManifest(self):
        "Reads and checks against signed source manifest"
        result, self.source_failed, self.source_missing = self.verifySignedManifest(self.ssmName)
        return result

    def compilePy(self, filename, force=0, verbose=0):
        "Recompile a python module if necessary."
        if verbose: print 'compilePy', filename, '...',
        cfile = filename + (__debug__ and 'c' or 'o')
        ftime = os.stat(filename)[stat.ST_MTIME]
        try: ctime = os.stat(cfile)[stat.ST_MTIME]
        except os.error: ctime = 0
        if (ctime > ftime) and not force:
            if verbose: print 'up-to-date'
            return
        else:
            py_compile.compile(filename)
            if verbose: print 'recompiled',cfile


    def compilePrep(self, filename, force=0, verbose=0):
        "Recompile a preppy module if necessary."
        # todo - harmonize print output and force with Python
        # modules
        preppy.compileModule(filename)

    def compileApp(self, force=0):
        "Default compilation for prep and py files in source list"
        for filename in self.getList():
            root, ext = os.path.splitext(filename)
            # cute - if ends in .py, look for compilePy;
            # if ends in spam, look for compileSpam. You
            # can subclass for new file types
            try:
                compileMethodName = 'compile' + ext[1].upper() + ext[2:]
                compileMethod = getattr(self, compileMethodName, None)
                if compileMethod:
                    compileMethod(filename, force=force)
            except:
                print "Can't compile '%s'" % filename

    def getCompiledFileName(self, filename):
        "What depends on this? Used to build target file list"
        # default is a 1-to-1 correspondence but we should
        # allow for one source having multiple outputs.
        # this returns either a single name or a list of names.
        root, ext = os.path.splitext(filename)
        if ext == '.py':
            compiled = filename + (__debug__ and 'c' or 'o')
        elif ext == '.prep':
            # I guess pyo files are a possibility...
            compiled = root + '.py' + (__debug__ and 'c' or 'o')
        else:
            compiled = filename
        return compiled

    def getTargetFileList(self):
        "Work out the list of filenames expected in the target installation"
        sourceFiles = self.getList()
        # we may get dependencies - one-to-many or many-to-one
        # so build a set of outputs and sort them at the end.

        targets = {}
        for sourceFile in sourceFiles:
            comp = self.getCompiledFileName(sourceFile)
            # it might return a string, or a list of more than one name
            if type(comp) == type(''):
                targets[comp] = 1
            else:  # got a list of stuff
                for outfile in comp:
                    targets[outfile] = 1

        # the default sort would mix files and directories;
        # a.text, b/subdir.txt, c.txt.  So sort in a 'pretty'
        # manner - files and directories.
        tmp = []
        T = targets.keys()
        if self._force_ssmName and self.ssmName not in T: T.append(self.ssmName)    #force add source manifest
        for filename in T:
            directory, shortname = os.path.split(filename)
            tmp.append((directory, shortname, filename))
        tmp.sort()
        outfiles = []
        for (a,b,c) in tmp:
            outfiles.append(c)

        return outfiles

    def writeSignedTargetManifest(self):
        "Generates signed target manifest for compiled application"
        self.writeSignedManifest(self.getTargetFileList(),self.stmName)

    def verifySignedTargetManifest(self):
        "Reads and checks against signed target manifest"
        result, self.target_failed, self.target_missing = self.verifySignedManifest(self.stmName)
        return result

    def findSourceFilesInTarget(self):
        "Hunt for uncompiled files e.g on a public web server"
        # intent is to find .py and .prep files.  Be cleverer -
        # - there will be complex dependency issues e.g. with
        # pagecatcher forms. The rule: if it is present in sourcefiles
        # and absent in target file list, assume it should not be there.
        strays = []

        sourceFiles = self.getList()
        targetFiles = self.getTargetFileList()

        targetTest = {} # hash for quick testing of presence
        for targetFile in targetFiles:
            targetTest[targetFile] = 1

        for srcFile in sourceFiles:
            if targetTest.has_key(srcFile):
                pass # OK, same in both versions e.g. HTML file
            else:
                if os.path.isfile(srcFile):
                    strays.append(srcFile)
        return strays

    def findUnexpectedFilesInTarget(self, exceptions=[]):
        """Finds any files apparently not from the app at all.
        Checks any directory for which there are any files listed.
        Subdirectories (logs, output etc.) are not scanned. Also
        you may give a list of known exceptions which are OK,
        for things the app generates locally."""
        unexpected = []
        doneDirs = {}
        sourceFiles = self.getList()
        targetFiles = self.getTargetFileList()

        # make some dictionaries for fast testing
        sourceSet = {}
        for src in sourceFiles:
            sourceSet[src] = 1
        targetSet = {}
        for targ in targetFiles:
            targetSet[targ] = 1

        for targetFileName in self.getTargetFileList():
            dirname, filename = os.path.split(targetFileName)
            if doneDirs.has_key(dirname):
                # we did it already, skip
                continue
            else:
                # scan for anything unknown in this directory
                doneDirs[dirname] = 1
                if dirname == '':
                    filelist = os.listdir('.')
                else:
                    filelist = os.listdir(dirname)

                for filename in filelist:
                    # windows will use 'parent\\subdir', but we want to
                    # test if it is in our list as 'parent/subdir'
                    normalized = string.replace(os.path.join(dirname, filename),
                                                os.sep, '/')
                    if filename in [self.ssmName, self.stmName]:
                        pass
                    elif sourceSet.has_key(normalized):
                        pass
                    elif targetSet.has_key(normalized):
                        pass
                    elif os.path.isdir(normalized):
                        pass
                    else:
                        unexpected.append(normalized)

        return unexpected

    def promoteTargetFiles(self, destDir, createDest=0):
        "Copies them up to destDir if changed"
        if not os.path.isdir(destDir):
            if createDest:
                os.makedirs(destDir)
            else:
                raise IOError, "destination directory '%s'not found, please check spelling!" % destDir

        for targetName in self.getTargetFileList():
            dirname, filename = os.path.split(targetName)
            if not os.path.isdir(os.path.join(destDir, dirname)):
                os.makedirs(os.path.join(destDir, dirname))
            shutil.copy2(os.path.join(dirname,filename), os.path.join(destDir, dirname))
            print 'copied',os.path.join(destDir, dirname, filename)

        # and finally the manifest
        shutil.copy2(self.stmName, destDir)

    def promoteSourceFiles(self, destDir, createDest=0):
        "Copies them up to destDir, preserving timestamps"
        if not os.path.isdir(destDir):
            if createDest:
                os.makedirs(destDir)
            else:
                raise IOError, "destination directory '%s'not found, please check spelling!" % destDir

        for targetName in self.getList():
            dirname, filename = os.path.split(targetName)
            if not os.path.isdir(os.path.join(destDir, dirname)):
                os.makedirs(os.path.join(destDir, dirname))
            shutil.copy2(os.path.join(dirname,filename), os.path.join(destDir, dirname))
            print 'copied',os.path.join(destDir, dirname, filename)

        # and finally the manifest
        shutil.copy2(self.ssmName, destDir)

    def promoteViaFTP(self):
        "Moves compiled code onto another box..."
        raise NotImplementedError

    def _writeArchive(self,arch,filelist,copyDir=None):
        if copyDir:
            from buildutils import copyTree, kill
            if os.path.isdir(copyDir): kill(copyDir)
            if self.verbose: print 'Copying to directory "%s"' % copyDir
            copyTree('.',copyDir,excludeSVN=1,exclude=[copyDir])
            filelist = map(lambda x, copyDir=copyDir: os.path.join(copyDir,x), filelist)
        for filename in filelist:
            arch.write(filename)
            if self.verbose: print 'archiving',filename
        if copyDir: kill(copyDir)
        arch.close()

    def makeArchive(self, archiveName, filelist, kind='zip', prefix=None, copyDir=None):
        """Create a zip or tgz containing given files.  Uses tarfile
        module from 3rd party which will hopefully go into Python 2.3 one
        day; thanks to author!"""
        if kind is None:
            # infer from extension.  .tar.gz is not handled by splitext;
            # check ourselves
            if archiveName[-4:] == '.zip':
                kind = 'zip'
            elif archiveName[-4:] == '.tar':
                kind = 'tar'
            elif archiveName[-4:] == '.tgz':
                kind = 'tgz'
            elif archiveName[-7:] == '.tar.gz':
                kind = 'tar.gz'
            elif archiveName[-7:] == '.pyz':
                kind = 'pyz'

        if kind in ['tgz', 'tar.gz']:
            # the tar module I found is Python 2.2 only and chokes
            # when gzipping; had to handle that myself
            from rlextra.utils import tarfile
            import tempfile
            tarName = tempfile.mktemp()
            arch = tarfile.TarFileCompat(tarName, 'w', compression=tarfile.TAR_PLAIN)
            self._writeArchive(arch,filelist,copyDir=copyDir)

            import gzip
            g = gzip.GzipFile(archiveName, 'wb', 9)
            f = open(tarName, 'rb')
            while 1:
                chunk = f.read(1024)
                if not chunk:
                    break
                g.write(chunk)
            g.close()
            f.close()
            os.remove(tarName)
        elif kind =='tar':
            from rlextra.utils import tarfile
            arch = tarfile.TarFileCompat(archiveName, 'w', compression=tarfile.TAR_PLAIN)
            self._writeArchive(arch,filelist,copyDir=copyDir)
        elif kind == 'zip':
            arch = zipfile.ZipFile(archiveName, 'w', compression=zipfile.ZIP_DEFLATED)
            self._writeArchive(arch,filelist,copyDir=copyDir)
        elif kind == 'pyz':
            # ensure installer on path
            import rlextra
            instDir = os.path.abspath(os.path.join(os.path.dirname(rlextra.__file__),'distro','tools','Installer'))
            print 'instDir = %s' % instDir
            if instDir not in sys.path:
                sys.path.insert(0,instDir)
            print 'added directory %s to path' % instDir
            from MEInc.Dist.archive import ZlibArchive
            from py_compile import compile
            instDir = os.path.join(instDir, 'support')
            # Z is the table of contents for the archive
            Z = []
            #by default add the unarchive code
            preLoad=1
            if preLoad:
                for n in ('archive_rt','imputil'):
                    f = os.path.join(instDir,n+'.py')
                    c = f+'c'
                    compile(f,c,f)
                    Z.append((n,c))
            for fileName in filelist:
                #work out the 'import name'
                nameMinusExt = os.path.splitext(fileName)[0]
                importName = string.replace(nameMinusExt, os.sep, '.')
                importName = string.replace(importName, '/', '.')
                print 'store %s as %s' % (fileName, importName)
                Z.append((importName, fileName))
            arch = ZlibArchive().build(archiveName, Z)
            #def makePYZ(where, pyz_exclude=(), remove_after_pyz=(), pyzName=None,
            # preLoad=1, addLib=1, cleanUp=1):

        else:
            raise ValueError, "unknown format, use 'zip' or 'tgz' or 'pyz' only"

        if self.verbose: print 'wrote',archiveName
        return os.stat(archiveName)[stat.ST_SIZE]

    def makeSourceArchive(self, archiveName, kind=None, copyDir=None):
        "Create a zip or tgz containing all source files"
        files = self.getList()
        files.append(self.ssmName)
        return self.makeArchive(archiveName, files, kind=kind, copyDir=copyDir)

    def makeTargetArchive(self, archiveName, kind=None):
        "Create a zip or tgz containing all compiled files"
        files = self.getTargetFileList()
        files.append(self.stmName)
        return self.makeArchive(archiveName, files, kind=kind)

    def targetHtmlReport(self, encrypted=1):
        "Verify a target installation"
        result = self.verifySignedTargetManifest()
        sources = self.findSourceFilesInTarget()
        unexp = self.findUnexpectedFilesInTarget()

        if result:
            print 'status:  ok'
        else:
            print 'status: bad', time.time()

        stuff = []
        L = stuff.append
        if len(self.target_failed):
            L('%d corrupted files:' % len(self.target_failed))
            for filename in self.target_failed:
                L('    ' + filename)
            L('')

        if len(self.target_missing):
            L('%d missing files:' % len(self.target_missing))
            for filename in self.target_missing:
                L('    ' + filename)
            L('')

        if len(sources):
            L('%d unexpected source files:' % len(sources))
            for filename in sources:
                L('    ' + filename)
            L('')

        if len(unexp):
            L('%d totally unexpected files:' % len(unexp))
            for filename in unexp:
                L('    ' + filename)
            L('')

        if stuff or not result:
            M = self._getSignedManifest(self.ssmName)
            if M:
                L('\n\nSource Manifest\n===============')
                map(L,M)

        if encrypted:
            print gloop.Gloop('hex').encode('\n'.join(stuff))
            print
        else:
            for line in stuff:
                print line

    def getCVSFilesUnderMe(self):
        """Returns a list of all CVS-committed files below directory"""
        filelist = []
        def walker(filelist, dirname, names):
            filelist.extend(getCVSFilesInDirectory(dirname))
        os.path.walk(self.baseDir, walker, filelist)
        # if they begin with .\ or ./, trim it
        cleaned = []
        for filename in filelist:
            if filename[0:2] == os.curdir + os.sep:
                filename = filename[2:]
            cleaned.append(filename)
        return cleaned

def test(id,D,verbose=1,ws=1,cs=1):
    import pprint
    #need to point this at a non-customer demo later
    a = AppMonitor(id,D,verbose)
    if ws:
        a.writeSignedSourceManifest()
        print
    if cs:
        print 'checking source_manifest...'
        a.verifySignedSourceManifest()
        if a.source_failed and verbose:
            print 'Failed'
            if verbose>1:
                pprint.pprint(a.source_failed)
    if verbose: print 'done'

if __name__=='__main__':
    import sys
    from reportlab import rl_config
# robin, apologies, dunno if mine is better but it helps
# test - you can call any method name from the command line
# directly and just point it at a directory.

    args = sys.argv[:]

    if '-v' in args:
        args.remove('-v')
        verbose = 1
    else:
        verbose = 0

    if len(args) < 2:
        print 'Usage:  appmonitor.py directory methodname [arg1] [arg2] [arg3]...'
        print 'n.b.    appmonitor.py ../myapp targetHtmlReport produces HTML'
    else:
        directory = args[1]
        if not os.path.isdir(directory):
            print 'directory %s not found, try again!' % directory
            sys.exit()

        methodName = args[2]
        if methodName == 'targetHtmlReport':
            print 'Content-type: text/html'
            print
            print '<html><head><title>%s</title></head><body><pre>' % methodName

        args = args[3:]
        a = AppMonitor(dirname=directory, verbose=verbose)
        methObj = getattr(a, methodName, None)
        if methObj is None:
            print 'command "%s" not found!' % methodName
        else:
            retVal = apply(methObj, args)
            if retVal is None:
                print 'Completed.'
            else:
                print ' Completed.  Return value:'
                pprint.pprint(retVal)
                if methodName[0:6] == 'verify' and retVal == 0:
                    print 'source_failed = %s' % a.source_failed
                    print 'source_missing = %s' % a.source_missing
                    print 'target_failed = %s' % a.target_failed
                    print 'target_missing = %s' % a.target_missing


        if methodName == 'targetHtmlReport':
            print '</pre></body></html>'
