| 1 | ##!/usr/bin/env python |
|---|
| 2 | ## -*- coding: utf8 -*- |
|---|
| 3 | """ |
|---|
| 4 | Sandbox for functions in developpment |
|---|
| 5 | |
|---|
| 6 | NOT FULLY TESTED! DON'T COMPLAIN IF YOUR COMPUTER EXPLODED! |
|---|
| 7 | |
|---|
| 8 | """ |
|---|
| 9 | import copy, os, numpy |
|---|
| 10 | from NeuroTools.parameters import ParameterSet |
|---|
| 11 | |
|---|
| 12 | def make_name(params_set,range_keys): |
|---|
| 13 | range_keys.sort() |
|---|
| 14 | name = '' |
|---|
| 15 | chars_to_be_removed = ['[',']','/',' ',':','(',')','{','}',',','.',"'"] |
|---|
| 16 | for key in range_keys: |
|---|
| 17 | if key is not None: |
|---|
| 18 | name = name + key + '_' + str(eval('params_set.'+key)) + '_' |
|---|
| 19 | for char in chars_to_be_removed: |
|---|
| 20 | name = name.replace(char,'') |
|---|
| 21 | return name |
|---|
| 22 | |
|---|
| 23 | def check_name(sim_name): |
|---|
| 24 | if os.path.exists(sim_name+'running'): |
|---|
| 25 | return False |
|---|
| 26 | elif not os.path.exists(sim_name+'running'): |
|---|
| 27 | os.system('touch '+sim_name+'running') |
|---|
| 28 | return True |
|---|
| 29 | |
|---|
| 30 | def string_table(tablestring): |
|---|
| 31 | """Convert a table written as a multi-line string into a dict of dicts.""" |
|---|
| 32 | tabledict = {} |
|---|
| 33 | rows = tablestring.strip().split('\n') |
|---|
| 34 | column_headers = rows[0].split() |
|---|
| 35 | for row in rows[1:]: |
|---|
| 36 | row = row.split() |
|---|
| 37 | row_header = row[0] |
|---|
| 38 | tabledict[row_header] = {} |
|---|
| 39 | for col_header,item in zip(column_headers[1:],row[1:]): |
|---|
| 40 | tabledict[row_header][col_header] = float(item) |
|---|
| 41 | return tabledict |
|---|
| 42 | |
|---|
| 43 | def string_table_ParameterSet(tablestring): |
|---|
| 44 | """Convert a table written as a multi-line string into a dict of dicts.""" |
|---|
| 45 | tabledict = ParameterSet({}) |
|---|
| 46 | rows = tablestring.strip().split('\n') |
|---|
| 47 | column_headers = rows[0].split() |
|---|
| 48 | for row in rows[1:]: |
|---|
| 49 | row = row.split() |
|---|
| 50 | row_header = row[0] |
|---|
| 51 | tabledict[row_header] = ParameterSet({}) |
|---|
| 52 | for col_header,item in zip(column_headers[1:],row[1:]): |
|---|
| 53 | tabledict[row_header][col_header] = float(item) |
|---|
| 54 | return tabledict |
|---|
| 55 | |
|---|
| 56 | def rUpdate(targetDict, itemDict): |
|---|
| 57 | for key, val in itemDict.items(): |
|---|
| 58 | if type(val) == type({}): |
|---|
| 59 | newTarget = targetDict.setdefault(key,{}) |
|---|
| 60 | rUpdate(newTarget, val) |
|---|
| 61 | else: |
|---|
| 62 | targetDict[key] = val |
|---|
| 63 | # ====================================================================== # |
|---|
| 64 | # creating experiments dict. |
|---|
| 65 | # ====================================================================== # |
|---|
| 66 | |
|---|
| 67 | def get_experiment_list(params): |
|---|
| 68 | """ |
|---|
| 69 | Takes params = dict with all parameters |
|---|
| 70 | Calculates cross product of all and returns a list with all experiments. |
|---|
| 71 | """ |
|---|
| 72 | f=lambda ss,row=[],level=0: len(ss)>1 \ |
|---|
| 73 | and reduce(lambda x,y:x+y,[f(ss[1:],row+[i],level+1) for i in ss[0]]) \ |
|---|
| 74 | or [row+[i] for i in ss[0]] |
|---|
| 75 | |
|---|
| 76 | tmplist=[] |
|---|
| 77 | |
|---|
| 78 | names = params.keys() |
|---|
| 79 | for experiment in f(params.values()): |
|---|
| 80 | tmptmpdict = {} |
|---|
| 81 | for name , value in zip(names,experiment): |
|---|
| 82 | tmptmpdict[name]=value |
|---|
| 83 | tmplist.append(tmptmpdict) |
|---|
| 84 | |
|---|
| 85 | return tmplist |
|---|
| 86 | |
|---|
| 87 | def get_experiment_dict(params): |
|---|
| 88 | """ |
|---|
| 89 | Takes params = dict with all parameters |
|---|
| 90 | Calculates cross product of all and returns a dict with all experiments. |
|---|
| 91 | """ |
|---|
| 92 | f=lambda ss,row=[],level=0: len(ss)>1 \ |
|---|
| 93 | and reduce(lambda x,y:x+y,[f(ss[1:],row+[i],level+1) for i in ss[0]]) \ |
|---|
| 94 | or [row+[i] for i in ss[0]] |
|---|
| 95 | |
|---|
| 96 | count = 0 |
|---|
| 97 | tmpdict={} |
|---|
| 98 | |
|---|
| 99 | names = params.keys() |
|---|
| 100 | for experiment in f(params.values()): |
|---|
| 101 | exp_name = 'exp' + str(count) |
|---|
| 102 | |
|---|
| 103 | tmptmpdict = {} |
|---|
| 104 | for name , value in zip(names,experiment): |
|---|
| 105 | |
|---|
| 106 | tmptmpdict[name]=value |
|---|
| 107 | tmpdict[exp_name] = tmptmpdict |
|---|
| 108 | count +=1 |
|---|
| 109 | return tmpdict |
|---|
| 110 | |
|---|
| 111 | def make_experiments(parameters,parameters_template, use_name = True): |
|---|
| 112 | experiments_tmp = get_experiment_dict(parameters) |
|---|
| 113 | experiments = {} |
|---|
| 114 | for i, experiment_tmp in enumerate(experiments_tmp.values()): |
|---|
| 115 | experiment = copy.deepcopy(parameters_template) |
|---|
| 116 | experiment['run'] = copy.deepcopy(experiment_tmp) |
|---|
| 117 | rUpdate(experiment,experiment_tmp) |
|---|
| 118 | experiments[i] = experiment |
|---|
| 119 | return experiments |
|---|
| 120 | |
|---|
| 121 | |
|---|
| 122 | # ====================================================================== # |
|---|
| 123 | def cross(*args): |
|---|
| 124 | """ |
|---|
| 125 | Return the cross-product of a variable number of lists (e.g. of a list |
|---|
| 126 | of lists). |
|---|
| 127 | |
|---|
| 128 | print cross(s1,s2,s3) |
|---|
| 129 | OBSOLETE / LESS EFFICIENT than get_experiment_dict |
|---|
| 130 | From: |
|---|
| 131 | http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/159975 |
|---|
| 132 | """ |
|---|
| 133 | |
|---|
| 134 | ans = [[]] |
|---|
| 135 | for arg in args: |
|---|
| 136 | ans = [x+[y] for x in ans for y in arg] |
|---|
| 137 | return ans |
|---|
| 138 | |
|---|
| 139 | |
|---|
| 140 | |
|---|
| 141 | def get_connectivity(params): |
|---|
| 142 | """ |
|---|
| 143 | |
|---|
| 144 | """ |
|---|
| 145 | a = params['a'] |
|---|
| 146 | radius = params['radius'] |
|---|
| 147 | radius_normalized = radius/a # when a is |
|---|
| 148 | population = params['population'] |
|---|
| 149 | |
|---|
| 150 | center = (int(population.dim[0]/2.),int(population.dim[1]/2.)) |
|---|
| 151 | offset = (int(round(population.dim[0]*radius_normalized)),int(round(population.dim[1]*radius_normalized))) |
|---|
| 152 | |
|---|
| 153 | targets={} |
|---|
| 154 | targets_gid={} |
|---|
| 155 | for n in range(center[0]-offset[0],center[0]+offset[0]+1): |
|---|
| 156 | for m in range(center[1]-offset[1],center[1]+offset[1]+1): |
|---|
| 157 | #print 'n: ',n, ' m: ',m |
|---|
| 158 | gid = population[n,m] |
|---|
| 159 | targets_tmp = pynest.getDict([gid])[0]['targets'] |
|---|
| 160 | targets_n_m = [] |
|---|
| 161 | for tgid in targets_tmp: |
|---|
| 162 | targets_n_m.append(population.locate(tgid)) |
|---|
| 163 | |
|---|
| 164 | targets[(n,m)]=targets_n_m |
|---|
| 165 | targets_gid[gid]=targets_tmp |
|---|
| 166 | return targets, targets_n_m |
|---|
| 167 | |
|---|
| 168 | |
|---|
| 169 | |
|---|
| 170 | |
|---|
| 171 | |
|---|
| 172 | def run_simulations(model,url,tag): |
|---|
| 173 | # TODO : this is not recommanded (SyntaxWarning) |
|---|
| 174 | from NeuroTools.benchmark import * |
|---|
| 175 | #lcn = LocalNetwork(0.1) |
|---|
| 176 | #lcn = model |
|---|
| 177 | print 'Simulations start' |
|---|
| 178 | print '######################################' |
|---|
| 179 | file = openHDF5File(url, "r") |
|---|
| 180 | data_root = file.getStructure(nodepath = "/", structure = True) |
|---|
| 181 | file.close() |
|---|
| 182 | if data_root.has_key('benchmark_finished'): |
|---|
| 183 | print 'Benchmark done. Data is in: ', url |
|---|
| 184 | return |
|---|
| 185 | |
|---|
| 186 | params = data_root['params'] |
|---|
| 187 | |
|---|
| 188 | |
|---|
| 189 | |
|---|
| 190 | experiments = data_root['run'].keys() |
|---|
| 191 | #finished = False |
|---|
| 192 | |
|---|
| 193 | # create a tmp dict with the name of the tag |
|---|
| 194 | root_dir = os.getcwd() |
|---|
| 195 | if not(os.path.exists('results/'+tag)): |
|---|
| 196 | os.mkdir('results/'+tag) |
|---|
| 197 | |
|---|
| 198 | # go there |
|---|
| 199 | os.chdir('results/'+tag) |
|---|
| 200 | |
|---|
| 201 | # check if this node is the copy machine at the end |
|---|
| 202 | # print 'check for copy node' |
|---|
| 203 | # copy_node = False |
|---|
| 204 | # if not(os.path.exists('final_copy_to_h5')): |
|---|
| 205 | # print 'I am the copy_node' |
|---|
| 206 | # copy_node = True |
|---|
| 207 | # os.system('touch final_copy_to_h5') |
|---|
| 208 | #else: |
|---|
| 209 | # print 'NO, I am not the copy_node' |
|---|
| 210 | |
|---|
| 211 | if data_root['run'][experiments[0]].has_key('useHardware'): |
|---|
| 212 | # check if have hardware |
|---|
| 213 | haveHardware = False |
|---|
| 214 | try: |
|---|
| 215 | import pyNN.fhws1v2 as pyNN_ |
|---|
| 216 | haveHardware = True |
|---|
| 217 | except ImportError: |
|---|
| 218 | haveHardware = False |
|---|
| 219 | |
|---|
| 220 | if not haveHardware: |
|---|
| 221 | print 'remove the hardware sim' |
|---|
| 222 | # we have to sort out the sims that use the hardware |
|---|
| 223 | experiments_tmp = [] |
|---|
| 224 | for param in data_root['run'].keys(): |
|---|
| 225 | if data_root['run'][param]['useHardware'] == False: |
|---|
| 226 | experiments_tmp.append(param) |
|---|
| 227 | experiments = experiments_tmp |
|---|
| 228 | |
|---|
| 229 | |
|---|
| 230 | #if data_root['run'][experiments[0]].has_key('analysed'): |
|---|
| 231 | |
|---|
| 232 | |
|---|
| 233 | model.params = params |
|---|
| 234 | for experiment in experiments: |
|---|
| 235 | if data_root['run'][experiment].has_key('analysed'): |
|---|
| 236 | continue |
|---|
| 237 | |
|---|
| 238 | name='' |
|---|
| 239 | for key in data_root['run'][experiment].keys(): |
|---|
| 240 | name = name+'_'+key+'_'+str(data_root['run'][experiment][key]) |
|---|
| 241 | |
|---|
| 242 | |
|---|
| 243 | print 'checking name: ',name |
|---|
| 244 | system_test_run=name+'_run' |
|---|
| 245 | system_touch_run='touch '+name+'_run' |
|---|
| 246 | system_touch_done='touch '+name+'_done' |
|---|
| 247 | |
|---|
| 248 | if os.path.exists(system_test_run): |
|---|
| 249 | continue |
|---|
| 250 | else: |
|---|
| 251 | os.system(system_touch_run) |
|---|
| 252 | |
|---|
| 253 | # while not(finished): |
|---|
| 254 | # experiment = experiments[int(numpy.floor(numpy.random.rand()*len(experiments)))] #TODO restrict to finished_experiments |
|---|
| 255 | # if not(data_root['run'][experiment].has_key('begin')):#'end')):# |
|---|
| 256 | # file = openHDF5File(url, "a") |
|---|
| 257 | # file.setStructure({'begin' : datetime.datetime.now().isoformat()}, "/run/" + experiment, createparents = True) |
|---|
| 258 | # file.close() |
|---|
| 259 | |
|---|
| 260 | #params = data_root['build'] |
|---|
| 261 | # update |
|---|
| 262 | model.params.update(data_root['run'][experiment]) |
|---|
| 263 | print '\nExperiment:' |
|---|
| 264 | print 'column_url', url |
|---|
| 265 | #print 'retina_url', lcn.params['retina_url'], '\n' |
|---|
| 266 | print 'Parameter to be simulated:' |
|---|
| 267 | for key in data_root['run'][experiment].keys(): |
|---|
| 268 | print key,':',data_root['run'][experiment][key] |
|---|
| 269 | |
|---|
| 270 | |
|---|
| 271 | model.params['name']=name |
|---|
| 272 | |
|---|
| 273 | |
|---|
| 274 | #out_DATA = m7_1.run_m7_1(params) |
|---|
| 275 | # run sim |
|---|
| 276 | model.build_(model.params) |
|---|
| 277 | # lcn.params['populations']=lcn.populations |
|---|
| 278 | model.run_(model.params) |
|---|
| 279 | #m7_1.run_m7_1(params) |
|---|
| 280 | # touch the name_done such that the script knows that it has finished that sim |
|---|
| 281 | os.system(system_touch_done) |
|---|
| 282 | |
|---|
| 283 | # Test if all sims have been simulated |
|---|
| 284 | exps = experiments[:] |
|---|
| 285 | #print 'exps ', exps |
|---|
| 286 | all_done = False |
|---|
| 287 | |
|---|
| 288 | #for ex in exps: |
|---|
| 289 | # print ex |
|---|
| 290 | |
|---|
| 291 | |
|---|
| 292 | for experiment in experiments: |
|---|
| 293 | print 'experiment ',experiment |
|---|
| 294 | name='' |
|---|
| 295 | for key in data_root['run'][experiment].keys(): |
|---|
| 296 | name = name+'_'+key+'_'+str(data_root['run'][experiment][key]) |
|---|
| 297 | system_test_done=name+'_done' |
|---|
| 298 | # print system_test_done |
|---|
| 299 | if os.path.exists(system_test_done): |
|---|
| 300 | exps.pop(exps.index(experiment)) |
|---|
| 301 | #print 'len exps', len(exps) |
|---|
| 302 | #print exps |
|---|
| 303 | if len(exps) == 0: |
|---|
| 304 | all_done = True |
|---|
| 305 | |
|---|
| 306 | print 'all_done: ', all_done |
|---|
| 307 | never_do=False |
|---|
| 308 | #finished = False |
|---|
| 309 | #if all_done: # until h5 is working properly, I will save data in gdf format, basta |
|---|
| 310 | if never_do: |
|---|
| 311 | exps = experiments[:] |
|---|
| 312 | print 'I copy now' |
|---|
| 313 | # while not(finished): |
|---|
| 314 | for experiment in experiments: |
|---|
| 315 | name='' |
|---|
| 316 | for key in data_root['run'][experiment].keys(): |
|---|
| 317 | name = name+'_'+key+'_'+str(data_root['run'][experiment][key]) |
|---|
| 318 | |
|---|
| 319 | #system_test_done=name+'_done' |
|---|
| 320 | # print system_test_done |
|---|
| 321 | # if os.path.exists(system_test_done): |
|---|
| 322 | #experiments.pop(experiments.index(experiment)) |
|---|
| 323 | # print 'pop experiment' |
|---|
| 324 | # update |
|---|
| 325 | params.update(data_root['run'][experiment]) |
|---|
| 326 | params.update({'name': name}) |
|---|
| 327 | # read data |
|---|
| 328 | #print lcn.populations |
|---|
| 329 | print params |
|---|
| 330 | out_DATA = model.get_data_(params) |
|---|
| 331 | #out_DATA = m7_1.return_data(params) |
|---|
| 332 | |
|---|
| 333 | branch = '/run/' + experiment |
|---|
| 334 | file = openHDF5File(url, "a") |
|---|
| 335 | |
|---|
| 336 | for pop in out_DATA: |
|---|
| 337 | file.createSpikeList(branch, pop, rows = out_DATA[pop], dt = data_root['params']['dt'], spec = 'reltime_id', ref = None) |
|---|
| 338 | |
|---|
| 339 | # file.setStructure({'end' : datetime.datetime.now().isoformat()}, "/run/" + experiment, createparents = True) |
|---|
| 340 | file.close() |
|---|
| 341 | exps.pop(exps.index(experiment)) |
|---|
| 342 | |
|---|
| 343 | |
|---|
| 344 | if len(exps) == 0: |
|---|
| 345 | #finished = True |
|---|
| 346 | file = openHDF5File(url, "a") |
|---|
| 347 | file.setStructure({'benchmark_finished':True}, "/", createparents = True) |
|---|
| 348 | #file.setStructure({'benchmark_finished' : datetime.datetime.now().isoformat()}, "/run/", createparents = True) |
|---|
| 349 | file.close() |
|---|
| 350 | os.system('touch finished_all') |
|---|
| 351 | |
|---|
| 352 | if os.path.exists('finished_all'): |
|---|
| 353 | print 'done all, data should be in: ', url |
|---|
| 354 | # now del all in that dir |
|---|
| 355 | files = os.listdir('../'+tag) |
|---|
| 356 | for file in files: |
|---|
| 357 | os.remove(file) |
|---|
| 358 | os.chdir(root_dir) |
|---|
| 359 | os.rmdir(tag) |
|---|
| 360 | |
|---|
| 361 | else: |
|---|
| 362 | print 'not all done, still simulating, or copying.' |
|---|
| 363 | |
|---|
| 364 | os.chdir(root_dir) |
|---|
