Changeset 414 for trunk/src/io.py

Timestamp:

08/05/09 10:12:04 (3 years ago)

Author:

pierre

Message:

Add one other dependency, and I know it is bad... Especially because I think that this should not be a dependency, but that it should be included in NeuroTools. There is a nice package, TableIO, that allows more efficient and fast array creation from large file text. It is much more efficient that numpy.loadtxt or others methods, so since NeuroTools often ends up in loading such big files, it is useful. I also add a NestFile? standard format in io.py for those that would nest files after stimulations not postprocessed by pyNN and that still want to load them and play with NeuroTools

Files:

• trunk/src/io.py (modified) (8 diffs)

trunk/src/io.py

@@ +1 @@
+# -*- coding: utf-8 -*-
 """
 NeuroTools.io
@@ -15 +16 @@
 StandardPickleFile - object used to manipulate pickle representation of NeuroTools objects (spikes or
                      analog signals)
+NestFile           - object used to manipulate raw NEST file that would not have been saved by pyNN
+                     (without headers)
 DataHandler        - object to establish the interface between NeuroTools.signals and NeuroTools.io
 
@@ -28 +31 @@
 import os, logging, cPickle, numpy
 DEFAULT_BUFFER_SIZE = -1
+HAVE_TABLEIO        = False #check_dependency('TableIO')
+
+if HAVE_TABLEIO:
+    import TableIO
+
 
 class FileHandler(object):
@@ -108 +116 @@
         """
         cmd = ''
-        f = open(self.filename, 'r')
+        f   = open(self.filename, 'r')
         for line in f.readlines():
             if line[0] == '#':
@@ -127 +135 @@
         if hasattr(object, "dt"):
             self.metadata['dt']     = object.dt
-
+        
     def __check_params(self, params):
         """
@@ -153 +161 @@
                 raise Exception("dims can not be infered while reading %s" %self.filename)
         return params
-            
+
     def get_data(self, sepchar = "\t", skipchar = "#"):
         """
         Load data from a text file and returns an array of the data
         """
-        try:
-            import TableIO
+        if HAVE_TABLEIO:
             data = numpy.fliplr(TableIO.readTableAsArray(self.filename, skipchar))
-        except ImportError:
-            myfile = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
+        else:
+            myfile   = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
             contents = myfile.readlines()
             myfile.close()
-            data = []
-            for line in contents:
-                if (line[0] != skipchar):
-                    line = line.strip().split(sepchar)
-                    id   = [int(float(line[-1]))]
-                    id.extend(map(float, line[0:-1]))
-                    data.append(id)
+            data   = []
+            header = True
+            idx    = 0
+            while header:
+                if contents[idx][0] != skipchar:
+                    header = False
+                    break
+                idx += 1
+            for i in xrange(idx, len(contents)):
+                line = contents[i].strip().split(sepchar)
+                id   = [float(line[-1])]
+                id  += map(float, line[0:-1])
+                data.append(id)
             logging.debug("Loaded %d lines of data from %s" % (len(data), self))
             data = numpy.array(data, numpy.float32)
         return data
-
-    #def get_data(self, sepchar = "\t", skipchar = "#"):
-        #return numpy.loadtxt(self.filename, dtype=numpy.float32)
-
+    
     def write(self, object):
         # can we write to the file more than once? In this case, should use seek, tell
@@ -191 +201 @@
 
     def read_spikes(self, params):
-        fileobj = open(self.filename, 'r', DEFAULT_BUFFER_SIZE)
         self.__read_metadata()
         p = self.__check_params(params)
         from NeuroTools.signals import spikes
-        return spikes.SpikeList(self.get_data(), p['id_list'], p['t_start'], p['t_stop'], p['dims'])
+        data = self.get_data()
+        return spikes.SpikeList(data, p['id_list'], p['t_start'], p['t_stop'], p['dims'])
 
     def read_analogs(self, type, params):
-        fileobj = open(self.filename, 'r', DEFAULT_BUFFER_SIZE)
         self.__read_metadata()
         p = self.__check_params(params)
@@ -258 +267 @@
     def read_spikes(self, params):
         fileobj = file(self.filename,"r")
-        object = cPickle.load(fileobj)
-        object = self.__reformat(params, object)
+        object  = cPickle.load(fileobj)
+        object  = self.__reformat(params, object)
         return object
         
     def read_analogs(self, type, params):
         return self.read_spikes(params)
+
+
+
+class NestFile(FileHandler):
+    
+    def __init__(self, filename, padding=0, with_time=False, with_gid=True):
+        self.filename  = filename
+        self.metadata  = {}
+        assert (padding >= 0) and (type(padding) == int), "ERROR ! padding should be a positive int"
+        self.padding   = padding
+        self.with_time = with_time
+        self.with_gid  = with_gid
+        self.standardtxtfile = StandardTextFile(filename) 
+    
+    def write(self, object):
+        """
+        Write the object to the file. 
+        
+        Examples:
+            >> handler.write(SpikeListObject)
+            >> handler.write(VmListObject)
+        """
+        return self.standardtxtfile.write(object)
+    
+    def __check_params(self, params):
+        """
+        Establish a control/completion/correction of the params to create an object by 
+        using comparison and data extracted from the metadata.
+        """
+        if 'dt' in params:
+            if params['dt'] is None and 'dt' in self.metadata:
+                logging.debug("dt is infered from the file header")
+                params['dt'] = self.metadata['dt']
+        if params['id_list'] is None:
+            print "WARNING: id_list will be infered based on active cells..."
+        elif isinstance(params['id_list'], int): # allows to just specify the number of neurons
+            params['id_list'] = range(params['id_list'])
+        elif not isinstance(params['id_list'], list):
+            raise Exception("id_list should be an int or a list !")
+        if params['dims'] is None:
+            if 'dimensions' in self.metadata:
+                params['dims'] = self.metadata['dimensions']
+            else:
+                raise Exception("dims can not be infered while reading %s" %self.filename)
+        return params
+    
+    def get_data(self, sepchar = "\t", skipchar = "#"):
+        """
+        Load data from a text file and returns a list of data
+        """
+        if HAVE_TABLEIO:
+            data = TableIO.readTableAsArray(self.filename, skipchar)
+        else:
+            myfile   = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
+            contents = myfile.readlines()
+            myfile.close()
+            data = []
+            header = True
+            idx    = 0
+            while header:
+                if contents[idx][0] != skipchar:
+                    header = False
+                    break
+                idx += 1
+            for i in xrange(idx, len(contents)):
+                line = contents[i].strip().split(sepchar)
+                id   = [float(line[0])]
+                id  += map(float, line[1:])
+                data.append(id)
+        return numpy.array(data)
+
+    def _fix_id_list(self, data, params):
+        print "All gids are shifted by padding", self.padding
+        data[:,0] = numpy.array(data[:,0], int) - self.padding
+        if params['id_list'] is None:
+            params['id_list'] = numpy.unique(data[:,0])
+        return data, params
+
+    def read_spikes(self, params):
+        """
+        Read a SpikeList object from a file and return the SpikeList object, created from the File and
+        from the additional params that may have been provided
+        
+        Examples:
+            >> params = {'id_list' : range(100), 't_stop' : 1000}
+            >> handler.read_spikes(params)
+                SpikeList Object (with params taken into account)
+        """
+        p = self.__check_params(params)
+        from NeuroTools import signals
+        data      = self.get_data()
+        data, p   = self._fix_id_list(data, p)
+        return signals.SpikeList(data, p['id_list'], p['t_start'], p['t_stop'], p['dims'])
+    
+    def read_analogs(self, type, params):
+        p       = self.__check_params(params)
+        data    = self.get_data()
+        data, p = self._fix_id_list(data, p)
+        from NeuroTools.signals import analogs
+        if type == "vm":
+            return analogs.VmList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
+        elif type == "current":
+            return analogs.CurrentList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
+        elif type == "conductance":
+            if len(data[0,:]) > 2:
+                g_exc = analogs.ConductanceList(data[:,[0,1]] , p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
+                g_inh = analogs.ConductanceList(data[:,[0,2]] , p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
+                return [g_exc, g_inh]
+            else:
+                return analogs.ConductanceList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
+