#Copyright ReportLab Europe Ltd. 2000-2004
#see license.txt for license details
#history www.reportlab.co.uk/rl-cgi/viewcvs.cgi/rlextra/utils/datamaker.py
import string
from types import ListType, TupleType, StringType, IntType
import re
_sql_select_re = re.compile(r'\s*select\s+(.+)\s+from\s+',re.I)
_id_re=re.compile(r'\b[a-zA-Z_]\w*\b\s*(?!\()')

def _checkIn(x,N):
    if type(N) is IntType: return N
    return x in N

class DataMaker:
    """class defining data production"""
    def __init__(self,**kw):
        self._data = []
        self.__dict__.update(kw)

    def connect(self, verbose):
        "Open file, database connection, Excel or whatever"
        self.verbose = verbose

    def close(self):
        "Close file, database connection, Excel or whatever"
        pass

    def fetchNextDataSet(self):
        "Return next block of data as nested list, or None if no more"
        if len(self._data) == 0:
            return None
        else:
            block = []
            data = self._data
            grpCol = getattr(self,'_groupingCol',self.groupingColumn)
            row = 0
            initialGroup = data[row][grpCol]
            while 1:
                block.append(data[0])
                data = data[1:]
                if len(data) == 0:
                    break
                if data[0][grpCol] <> initialGroup:
                    break
            self._data = data
            return block

    def fetchData(self):
        "return a list of data sets"
        D = []
        a = D.append
        while 1:
            d = self.fetchNextDataSet()
            if d is None: return D
            a(d)

class CSVDataMaker(DataMaker):
    """Defines how to make data from a CSV file"""
    filename =  None        # name of the file or perhaps something with a readlines method
    headerRows = 1          # number of header rows
    nameRow = 0             # nameRow which row defines column names
    columnNames = None      # use if you need fake column names and headerRows = 0
    columnCount = None      # number of columns
    groupingColumn = 0      # which column to group by
    integerColumns = ()     # which columns are integer
    dateColumns = ()        # which columns are dates
    floatColumns = ()       # which columns are floats
    sep = ','               # what is the csv separator
    fast = 0                # use fast parsing ie assume that sep will work fine
    na = ['None', 'N/A']    # the not availble string
    sbl = 1                 # if blank lines should be stripped
    sql = None              # the sql to be used for calcs (we only handle real simple stuff)
    dateParser = None       # the date parser or None 
    no_na_dates = None      # if dates must be valid, either None, 0/1 or a sequence
                            # of date columns which must be checked

    def __init__(self,**kw):
        DataMaker.__init__(self,**kw)

    class _arithNone:
        def __add__(self,other):
            return self
        __sub__ = __mul__ = __div__ = __pow__ = __neg__ = __abs__ = __rsub__ = __rdiv__ = __add__
    _arithNone = _arithNone()

    def parseDate(self, text):
        """Attempt to get a date out.

          Currently only supports default Excel/CSV format of 31/01/1999 00:00:00
          and ignores hours/minutes."""
        daytext = string.split(text,' ')[0] # chop off hours if present
        d, m, y = map(int, string.split(daytext,'/'))
        if y < 100:
            if y > 80:
                y = y + 1900
            else:
                y = y + 2000
        from reportlab.lib.normalDate import NormalDate
        return NormalDate((y, m, d))

    def connect(self,verbose=0):
        from rlextra.utils.csv import _processLine
        lines = (hasattr(self.filename,'readlines') and self.filename or open(self.filename,'r')).readlines()
        if self.sbl: lines = filter(lambda x: string.strip(x),lines)

        n = self.headerRows
        if n>0:
            headers = lines[:n]
            lines = lines[n:]
            j = self.nameRow
            if j is not None:
                if j>=n: raise ValueError, "nameRow=%d not in headerRows [0:%d]" % (j,n)
                self._names = names = map(string.strip,map(string.lower,_processLine(headers[j],self.sep)))
        else:
            names = getattr(self,'columnNames',None)
            if names: self._names = names = map(string.strip,map(string.lower,names))

        sql = self.sql
        if callable(sql): sql = sql()
        self._groupingCol = self.groupingColumn
        if sql:
            if not names: raise ValueError, "sql attribute and no column names"
            try:
                self._checkCSVColumnNames('groupingColumn',names)
                self._groupingCol = names.index(self.groupingColumn)
            except:
                pass
            self._checkCSVColumnNames('integerColumns',names)
            self._checkCSVColumnNames('floatColumns',names)
            self._checkCSVColumnNames('dateColumns',names)
            m = _sql_select_re.match(sql)
            if not m: raise ValueError,"can't match %s.sql='%s'" % (self.__class__.__name__,sql)
            X = string.lower(string.strip(m.group(1)))
            if X!='*':
                sqlIDs = self._sqlIDs = map(string.strip,_id_re.findall(X))
                self._checkCSVColumnNames('_sqlIDs',names)
                E = []
                def handleID(m,names=names):
                    c = string.strip(m.group(0))
                    if c in names:
                        x = names.index(c)
                        return "self._exConvertField(_f,%d,'%s',lineNo)" % (x,c)
                    else:
                        return c

                for c in string.split(X,','):
                    c = string.strip(c)
                    if c in names: v = names.index(c)
                    else: v = _id_re.sub(handleID,c)
                    E.append((c,v))
            else:
                E = zip(names,range(len(names)))

        self._na = map(string.lower,self.na)+['']

        lineNo = 1
        rows = []
        for line in lines:
            if self.fast:
                _f = string.split(line,self.sep)
            else:
                _f = _processLine(line,self.sep)
            lineNo = lineNo + 1
            row = []

            if sql:
                for c,j in E:
                    if type(j) is IntType:
                        if j<len(_f):
                            v = self._convertField(_f[j],c,lineNo)
                        else:
                            v = None
                    else:
                        try:
                            v = eval(j,locals(),globals())
                            if v is self._arithNone: v = None
                        except:
                            raise ValueError, "Can't evaluate %s in %s line %d" % (c,self.filename,lineNo)
                    row.append(v)
            else:
                cols = self.columnCount
                if cols is None:
                    cols = getattr(self,'_names',None)
                    if cols: cols = len(cols)
                    else: cols = len(_f)
                    self.columnCount = cols
                else:
                    assert len(_f) <= cols, "Incorrect number of fields at line %d in %s" % (lineNo, self.filename)
                    # we can have missing columns on the right, and they get padded with empty strings
                    while len(_f) < cols: _f.append('')
                for j in xrange(cols):
                    row.append(self._convertField(_f[j],j,lineNo))

            rows.append(row)

        self._data = rows
        del self._na

    def _checkCSVColumnNames(self,aName,names):
        A = getattr(self,aName,None)
        if A is None: return
        a = type(A)
        _A = not a in (ListType,TupleType)
        if _A:
            A = [A,]
            fnc = None
        else:
            fnc = a is ListType and list or tuple
            A = list(A)
        for i in xrange(len(A)):
            a = A[i]
            if type(a) is not StringType:
                raise ValueError, "%s instance has sql, but attribute %s contains non-string '%s'" % (self.__class__.__name__,aName,str(a))
            A[i] = a = string.lower(a)
            if a not in names:
                raise ValueError, "%s instance has sql, but attribute %s contains '%s' not in column names" % (self.__class__.__name__,aName,a)
        if fnc: A = fnc(A)
        else: A = A[0]
        setattr(self,aName,A)

    def _exConvertField(self,f,j,x,n):
        if j>=len(f): return self._arithNone
        r = self._convertField(f[j],x,n)
        if r is None: return self._arithNone
        return r

    def _convertField(self,f,x,n):
        try:
            typ = 'str'
            #accept nulls
            if string.lower(f) in self._na:
                no_na_dates = self.no_na_dates
                if no_na_dates and x in self.dateColumns and _checkIn(x,no_na_dates):
                    raise ValueError('Bad date field')
                return None
            elif x in self.integerColumns:
                typ = 'int'
                return int(f)
            elif x in self.floatColumns:
                typ = 'float'
                return float(f)
            elif x in self.dateColumns:
                typ = 'date'
                return (self.dateParser or self.parseDate)(f)
            else:
                return f
        except:
            import traceback
            traceback.print_exc()
            raise ValueError, "Bad field at line %d field %s, expected %s: '%s'" % (n, str(x), typ, f)

class CSVReader:
    def __init__(self,filename,lineEndings=['\r\n','\r','\n\r']):
        self.filename = filename
        self.lineEndings = lineEndings

    def readlines(self):
        filename = self.filename
        data = (hasattr(filename,'read') and filename or open(filename,'rb')).read()
        for le in self.lineEndings:
            data = data.replace(le,'\n')
        return data.split('\n')

if __name__=='__main__':
    from reportlab.lib.utils import getStringIO
    f = getStringIO('''chartId\tFundId\tReturn\tVolatility
119\t1\t0.303739275\t0.045788029
119\t2\t0.340259329\t0.056684527
119\t3\t0.244538056\t0.044776268
119\t4\t0.379326509\t0.05526936
119\t5\t0.269164078\t0.048254073
120\t1\t0.212006856\t0.045515668
120\t2\t0.404000212\t0.049965404
120\t3\t0.416953391\t0.050843349
120\t4\t0.451333469\t0.040626584
666\t1\t0.417259534\t0.051285696
666\t2\t0.21576762\t0.047812899
666\t3\t0.420633734\t0.040486482
666\t4\t0.22950049\t0.059180818
666\t5\t0.485586939\t0.047515184
''')
    dm = CSVDataMaker(
                    filename=f,
                    sep='\t',
                    integerColumns=['chartId'],
                    floatColumns=['Return','Volatility'],
                    sql='SELECT chartId, 100*Return, 100*Volatility FROM scatterplot_data',
                    groupingColumn=0,
                    )
    dm.connect()
    r = 0
    while 1:
        fetched = dm.fetchNextDataSet()
        if not fetched: break
        r = r+1
        print 'Dataset',r
        for d in fetched:
            print d
