root/trunk/src/io.py

Revision 477, 20.4 KB (checked in by mschmucker, 16 months ago)

adapt to id_list now being a property (and not a function anymore).

Line 
1# -*- coding: utf-8 -*-
2"""
3NeuroTools.io
4==================
5
6A collection of functions to handle all the inputs/outputs of the NeuroTools.signals
7file, used by the loaders.
8
9Classes
10-------
11
12FileHandler        - abstract class which should be overriden, managing how a file will load/write
13                     its data
14StandardTextFile   - object used to manipulate text representation of NeuroTools objects (spikes or
15                     analog signals)
16StandardPickleFile - object used to manipulate pickle representation of NeuroTools objects (spikes or
17                     analog signals)
18NestFile           - object used to manipulate raw NEST file that would not have been saved by pyNN
19                     (without headers)
20DataHandler        - object to establish the interface between NeuroTools.signals and NeuroTools.io
21
22All those objects can be used with NeuroTools.signals
23
24    >> data = StandardTextFile("my_data.dat")
25    >> spikes = load(data,'s')
26"""
27
28
29from NeuroTools import check_dependency
30
31import os, logging, cPickle, numpy
32DEFAULT_BUFFER_SIZE = -1
33HAVE_TABLEIO        = check_dependency('TableIO')
34
35if HAVE_TABLEIO:
36    import TableIO
37
38
39class FileHandler(object):
40    """
41    Class to handle all the file read/write methods for the key objects of the
42    signals class, i.e SpikeList and AnalogSignalList. Could be extented
43   
44    This is an abstract class that will be implemented for each format (txt, pickle, hdf5)
45    The key methods of the class are:
46        write(object)              - Write an object to a file
47        read_spikes(params)        - Read a SpikeList file with some params
48        read_analogs(type, params) - Read an AnalogSignalList of type `type` with some params
49   
50    Inputs:
51        filename - the file name for reading/writing data
52   
53    If you want to implement your own file format, you just have to create an object that will
54    inherit from this FileHandler class and implement the previous functions. See io.py for more
55    details
56    """
57   
58    def __init__(self, filename):
59        self.filename = filename
60   
61    def __str__(self):
62        return "%s (%s)" % (self.__class__.__name__, self.filename)
63   
64    def write(self, object):
65        """
66        Write the object to the file.
67       
68        Examples:
69            >> handler.write(SpikeListObject)
70            >> handler.write(VmListObject)
71        """
72        return _abstract_method(self)
73   
74    def read_spikes(self, params):
75        """
76        Read a SpikeList object from a file and return the SpikeList object, created from the File and
77        from the additional params that may have been provided
78       
79        Examples:
80            >> params = {'id_list' : range(100), 't_stop' : 1000}
81            >> handler.read_spikes(params)
82                SpikeList Object (with params taken into account)
83        """
84        return _abstract_method(self)
85   
86    def read_analogs(self, type, params):
87        """
88        Read an AnalogSignalList object from a file and return the AnalogSignalList object of type
89        `type`, created from the File and from the additional params that may have been provided
90       
91        `type` can be in ["vm", "current", "conductance"]
92       
93        Examples:
94            >> params = {'id_list' : range(100), 't_stop' : 1000}
95            >> handler.read_analogs("vm", params)
96                VmList Object (with params taken into account)
97            >> handler.read_analogs("current", params)
98                CurrentList Object (with params taken into account)
99        """
100        if not type in ["vm", "current", "conductance"]:
101            raise Exception("The type %s is not available for the Analogs Signals" %type)
102        return _abstract_method(self)
103
104
105
106class StandardTextFile(FileHandler):
107   
108    def __init__(self, filename):
109        FileHandler.__init__(self, filename)
110        self.metadata = {}
111
112    def __read_metadata(self):
113        """
114        Read the informations that may be contained in the header of
115        the NeuroTools object, if saved in a text file
116        """
117        cmd = ''
118        variable = None
119        label    = None
120        f   = open(self.filename, 'r')
121        for line in f.readlines():
122            if line[0] == '#':
123                if line[1:].strip().find('variable') != -1:
124                  variable = line[1:].strip().split(" = ")
125                elif line[1:].strip().find('label') != -1: 
126                  label = line[1:].strip().split(" = ")                 
127                else:
128                  cmd += line[1:].strip() + ';'
129            else:
130                break
131        f.close()
132        exec cmd in None, self.metadata
133        if not variable is None:
134          self.metadata[variable[0]] = variable[1]
135        if not variable is None:
136          self.metadata[label[0]] = label[1]
137       
138
139    def __fill_metadata(self, object):
140        """
141        Fill the metadata from those of a NeuroTools object before writing the object
142        """
143        self.metadata['dimensions'] = str(object.dimensions)
144        if len(object.id_list > 0):
145            self.metadata['first_id'] = numpy.min(object.id_list)
146            self.metadata['last_id']  = numpy.max(object.id_list)
147        if hasattr(object, "dt"):
148            self.metadata['dt']     = object.dt
149       
150    def __check_params(self, params):
151        """
152        Establish a control/completion/correction of the params to create an object by
153        using comparison and data extracted from the metadata.
154        """
155        if 'dt' in params:
156            if params['dt'] is None and 'dt' in self.metadata:
157                logging.debug("dt is infered from the file header")
158                params['dt'] = self.metadata['dt']
159        if not ('id_list' in params) or (params['id_list'] is None):
160            if ('first_id' in self.metadata) and ('last_id' in self.metadata):
161                params['id_list'] = range(int(self.metadata['first_id']), int(self.metadata['last_id'])+1)
162                logging.debug("id_list (%d...%d) is infered from the file header" % (int(self.metadata['first_id']), int(self.metadata['last_id'])+1))
163            else:
164                raise Exception("id_list can not be infered while reading %s" %self.filename)
165        elif isinstance(params['id_list'], int): # allows to just specify the number of neurons
166            params['id_list'] = range(params['id_list'])
167        elif not isinstance(params['id_list'], list):
168            raise Exception("id_list should be an int or a list !")
169        if not ('dims' in params) or (params['dims'] is None):
170            if 'dimensions' in self.metadata:
171                params['dims'] = self.metadata['dimensions']
172            else:
173                params['dims'] = len(params['id_list'])
174        return params
175
176    def get_data(self, sepchar = "\t", skipchar = "#"):
177        """
178        Load data from a text file and returns an array of the data
179        """
180        if HAVE_TABLEIO:
181            data = numpy.fliplr(TableIO.readTableAsArray(self.filename, skipchar))
182        else:
183            myfile   = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
184            contents = myfile.readlines()
185            myfile.close()
186            data   = []
187            header = True
188            idx    = 0
189            while header and idx < len(contents):
190                if contents[idx][0] != skipchar:
191                    header = False
192                    break
193                idx += 1
194            for i in xrange(idx, len(contents)):
195                line = contents[i].strip().split(sepchar)
196                id   = [float(line[-1])]
197                id  += map(float, line[0:-1])
198                data.append(id)
199            logging.debug("Loaded %d lines of data from %s" % (len(data), self))
200            data = numpy.array(data, numpy.float32)
201        return data
202   
203    def write(self, object):
204        # can we write to the file more than once? In this case, should use seek, tell
205        # to always put the header information at the top?
206        # write header
207        self.__fill_metadata(object)
208        fileobj = open(self.filename, 'w', DEFAULT_BUFFER_SIZE)
209        header_lines = ["# %s = %s" % item for item in self.metadata.items()]
210        fileobj.write("\n".join(header_lines) + '\n')
211        numpy.savetxt(fileobj, object.raw_data(), fmt = '%g', delimiter='\t')
212        fileobj.close()
213
214    def read_spikes(self, params):
215        self.__read_metadata()
216        p    = self.__check_params(params)
217        from NeuroTools.signals import spikes
218        data   = self.get_data()
219        result = spikes.SpikeList(data, p['id_list'], p['t_start'], p['t_stop'], p['dims'])
220        del data
221        return result
222
223    def read_analogs(self, type, params):
224        self.__read_metadata()
225        p = self.__check_params(params)
226        from NeuroTools.signals import analogs
227        if type == "vm":
228            return analogs.VmList(self.get_data(), p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
229        elif type == "current":
230            return analogs.CurrentList(self.get_data(), p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
231        elif type == "conductance":
232            data  = numpy.array(self.get_data())
233            if len(data[0,:]) > 2:
234                g_exc = analogs.ConductanceList(data[:,[0,1]] , p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
235                g_inh = analogs.ConductanceList(data[:,[0,2]] , p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
236                return [g_exc, g_inh]
237            else:
238                return analogs.ConductanceList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
239
240
241class StandardPickleFile(FileHandler):
242   
243    def __init__(self, filename):
244        FileHandler.__init__(self, filename) 
245        self.metadata = {}
246
247    def __fill_metadata(self, object):
248        """
249        Fill the metadata from those of a NeuroTools object before writing the object
250        """
251        self.metadata['dimensions'] = str(object.dimensions)
252        self.metadata['first_id']   = numpy.min(object.id_list)
253        self.metadata['last_id']    = numpy.max(object.id_list)
254        if hasattr(object, 'dt'):
255            self.metadata['dt']     = object.dt
256
257    def __reformat(self, params, object):
258        self.__fill_metadata(object)
259        if 'id_list' in params and params['id_list'] != None:
260            if isinstance(params['id_list'], int): # allows to just specify the number of neurons
261                params['id_list'] = range(params['id_list'])
262            if params['id_list'] != range(int(self.metadata['first_id']), int(self.metadata['last_id'])+1):
263                object = object.id_slice(params['id_list'])
264        do_slice = False
265        t_start = object.t_start
266        t_stop  = object.t_stop
267        if 't_start' in params and params['t_start'] is not None and params['t_start'] != object.t_start:
268            t_start = params['t_start']
269            do_slice = True
270        if 't_stop' in params and params['t_stop'] is not None and params['t_stop'] != object.t_stop:
271            t_stop = params['t_stop']
272            do_slice = True
273        if do_slice:
274            object = object.time_slice(t_start, t_stop)
275        return object
276   
277    def write(self, object):
278        fileobj = file(self.filename,"w")
279        return cPickle.dump(object, fileobj)
280
281    def read_spikes(self, params):
282        fileobj = file(self.filename,"r")
283        object  = cPickle.load(fileobj)
284        object  = self.__reformat(params, object)
285        return object
286       
287    def read_analogs(self, type, params):
288        return self.read_spikes(params)
289
290
291
292class NestFile(FileHandler):
293   
294    def __init__(self, filename, padding=0, with_time=False, with_gid=True):
295        self.filename  = filename
296        self.metadata  = {}
297        assert (padding >= 0) and (type(padding) == int), "ERROR ! padding should be a positive int"
298        self.padding   = padding
299        self.with_time = with_time
300        self.with_gid  = with_gid
301        self.standardtxtfile = StandardTextFile(filename) 
302   
303    def write(self, object):
304        """
305        Write the object to the file.
306       
307        Examples:
308            >> handler.write(SpikeListObject)
309            >> handler.write(VmListObject)
310        """
311        return self.standardtxtfile.write(object)
312   
313    def __check_params(self, params):
314        """
315        Establish a control/completion/correction of the params to create an object by
316        using comparison and data extracted from the metadata.
317        """
318        if 'dt' in params:
319            if params['dt'] is None and 'dt' in self.metadata:
320                logging.debug("dt is infered from the file header")
321                params['dt'] = self.metadata['dt']
322        if params['id_list'] is None:
323            print "WARNING: id_list will be infered based on active cells..."
324        elif isinstance(params['id_list'], int): # allows to just specify the number of neurons
325            params['id_list'] = range(params['id_list'])
326        elif not isinstance(params['id_list'], list):
327            raise Exception("id_list should be an int or a list !")
328        if params['dims'] is None:
329            if 'dimensions' in self.metadata:
330                params['dims'] = self.metadata['dimensions']
331            else:
332                raise Exception("dims can not be infered while reading %s" %self.filename)
333        return params
334   
335    def get_data(self, sepchar = "\t", skipchar = "#"):
336        """
337        Load data from a text file and returns a list of data
338        """
339        if HAVE_TABLEIO:
340            data = TableIO.readTableAsArray(self.filename, skipchar)
341        else:
342            myfile   = open(self.filename, "r", DEFAULT_BUFFER_SIZE)
343            contents = myfile.readlines()
344            myfile.close()
345            data = []
346            header = True
347            idx    = 0
348            while header:
349                if contents[idx][0] != skipchar:
350                    header = False
351                    break
352                idx += 1
353            for i in xrange(idx, len(contents)):
354                line = contents[i].strip().split(sepchar)
355                id   = [float(line[0])]
356                id  += map(float, line[1:])
357                data.append(id)
358        return numpy.array(data)
359
360    def _fix_id_list(self, data, params):
361        print "All gids are shifted by padding", self.padding
362        data[:,0] = numpy.array(data[:,0], int) - self.padding
363        if params['id_list'] is None:
364            params['id_list'] = numpy.unique(data[:,0])
365        return data, params
366
367    def read_spikes(self, params):
368        """
369        Read a SpikeList object from a file and return the SpikeList object, created from the File and
370        from the additional params that may have been provided
371       
372        Examples:
373            >> params = {'id_list' : range(100), 't_stop' : 1000}
374            >> handler.read_spikes(params)
375                SpikeList Object (with params taken into account)
376        """
377        p = self.__check_params(params)
378        from NeuroTools import signals
379        data      = self.get_data()
380        data, p   = self._fix_id_list(data, p)
381        return signals.SpikeList(data, p['id_list'], p['t_start'], p['t_stop'], p['dims'])
382   
383    def read_analogs(self, type, params):
384        p       = self.__check_params(params)
385        data    = self.get_data()
386        data, p = self._fix_id_list(data, p)
387        from NeuroTools.signals import analogs
388        if type == "vm":
389            return analogs.VmList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
390        elif type == "current":
391            return analogs.CurrentList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
392        elif type == "conductance":
393            if len(data[0,:]) > 2:
394                g_exc = analogs.ConductanceList(data[:,[0,1]] , p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
395                g_inh = analogs.ConductanceList(data[:,[0,2]] , p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
396                return [g_exc, g_inh]
397            else:
398                return analogs.ConductanceList(data, p['id_list'], p['dt'], p['t_start'], p['t_stop'], p['dims'])
399
400
401class PyNNNumpyBinaryFile(FileHandler):
402   
403    def __init__(self, filename):
404        FileHandler.__init__(self, filename)
405        self.fileobj = open(self.filename, 'r')
406       
407    def read_spikes(self, params):
408        from NeuroTools.signals import spikes
409        contents = numpy.load(self.fileobj)
410        spike_data = contents['data'][:, (1,0)]
411        self.metadata = M = {}
412        for k,v in contents['metadata']:
413            M[k] = eval(v)
414        id_list = range(M['first_id'], M['last_id'])
415        t_stop = params['t_stop']
416        # really need to check the agreement between file metadata and
417        # params for all metadata items
418        return spikes.SpikeList(spike_data, id_list, t_start=0.0, t_stop=t_stop,
419                                dims=M['dimensions'])
420       
421    #def read_analogs(self, type, params):
422    #    contents = numpy.load(self.fileobj)
423    #    values, ids = contents['data'].T # need to check the shape first
424   
425
426
427class DataHandler(object):
428    """
429    Class to establish the interface for loading/saving objects in NeuroTools
430   
431    Inputs:
432        filename - the user file for reading/writing data. By default, if this is
433                   string, a StandardTextFile is created
434        object   - the object to be saved. Could be a SpikeList or an AnalogSignalList
435       
436    Examples:
437        >> txtfile = StandardTextFile("results.dat")
438        >> DataHandler(txtfile)
439        >> picklefile = StandardPickelFile("results.dat")
440        >> DataHandler(picklefile)
441       
442    """
443    def __init__(self, user_file, object = None):
444        if type(user_file) == str:
445            user_file = StandardTextFile(user_file)
446        elif not isinstance(user_file, FileHandler):
447            raise Exception ("The user_file object should be a string or herits from FileHandler !")
448        self.user_file     = user_file
449        self.object        = object
450
451    def load_spikes(self, **params):
452        """
453        Function to load a SpikeList object from a file. The data type is automatically
454        infered. Return a SpikeList object
455       
456        Inputs:
457            params - a dictionnary with all the parameters used by the SpikeList constructor
458       
459        Examples:
460            >> params = {'id_list' : range(100), 't_stop' : 1000}
461            >> handler.load_spikes(params)
462                SpikeList object
463       
464        See also
465            SpikeList, load_analogs
466        """
467       
468        ### Here we should have an automatic selection of the correct manager
469        ### acccording to the file format.
470        ### For the moment, we try the pickle format, and if not working
471        ### we assume this is a text file
472        logging.debug("Loading spikes from %s, with parameters %s" % (self.user_file, params))
473        return self.user_file.read_spikes(params)
474
475
476    def load_analogs(self, type, **params):
477        """
478        Read an AnalogSignalList object from a file and return the AnalogSignalList object of type
479        `type`, created from the File and from the additional params that may have been provided
480       
481        `type` can be in ["vm", "current", "conductance"]
482       
483        Examples:
484            >> params = {'id_list' : range(100), 't_stop' : 1000}
485            >> handler.load_analogs("vm", params)
486                VmList Object (with params taken into account)
487            >> handler.load_analogs("current", params)
488                CurrentList Object (with params taken into account)
489       
490        See also
491            AnalogSignalList, load_spikes
492        """
493        ### Here we should have an automatic selection of the correct manager
494        ### acccording to the file format.
495        ### For the moment, we try the pickle format, and if not working
496        ### we assume this is a text file
497        logging.debug("Loading analog signal of type '%s' from %s, with parameters %s" % (type, self.user_file, params))
498        return self.user_file.read_analogs(type, params)
499
500
501    def save(self):
502        """
503        Save the object defined in self.object with the method os self.user_file
504       
505        Note that you can add your own format for I/O of such NeuroTools objects
506        """
507        ### Here, you could add your own format if you have created the appropriate
508        ### manager.
509        ### The methods of the manager are quite simple: should just inherits from the FileHandler
510        ### class and have read() / write() methods
511        if self.object == None:
512            raise Exception("No object has been defined to be saved !")
513        else:
514            self.user_file.write(self.object)
Note: See TracBrowser for help on using the browser.