Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

# encoding: utf-8 

from __future__ import print_function, division, absolute_import 

 

import contextlib 

import os 

from os.path import join, exists 

import shutil 

import tempfile 

 

from .database import setup_db, setup_fresh_db, check_if_tables_exist, copy_db 

from .errors import InvalidOperationError, InvalidLandingZone 

from .landing_zone import LandingZone 

from .logger import setup_logger_from, drop_logger, check_logging_config, logger 

from .utils import abs_folder, enumerate_filename, print_signals 

 

 

@contextlib.contextmanager 

def get_cmdline_logger(verbose): 

config_file = join(abs_folder(__file__), "cmdline_logging.yaml") 

setup_logger_from(config_file) 

if verbose: 

logger().setLevel(10) 

yield 

drop_logger() 

 

 

def init_config(landing_zone, reset, print_ok, print_err, verbose=False): 

"""setup minimal landing zone and create default configuration 

""" 

from datapool.config_handling import init_config 

with get_cmdline_logger(verbose): 

 

if exists(landing_zone): 

print_err("+ folder/file {} already exists".format(landing_zone)) 

return 1 

print_ok("- guess settings") 

try: 

config_folder, messages = init_config(landing_zone, overwrite=reset) 

for message in messages: 

print_err(" - {}".format(message)) 

print_ok("- created config files at {}".format(config_folder)) 

print_ok( 

" please edit these files and adapt the data base configuration to your setup") 

except Exception as e: 

print_err("+ something went wrong: {}".format(e)) 

return 1 

 

try: 

LandingZone(landing_zone).create_empty() 

print_ok("+ initialized landing zone at {}".format(landing_zone), fg="green") 

except Exception as e: 

print_err("+ something went wrong: {}".format(e)) 

return 1 

 

return 0 

 

 

def check_config(print_ok, print_err, verbose=False): 

"""checks if current configuration is valid, eg if database access is possible, or 

if matlab can be started. 

""" 

from datapool.config_handling import read_config, check_config 

config = read_config() 

 

if config is None: 

print_err("- no config file found. please run 'pool init-config' first.") 

return 1 

 

print_ok("- check settings in config file {}".format(config.__file__)) 

try: 

check_logging_config(config) 

except Exception as e: 

print_err("- could not setup logger. hint: {}".format(e)) 

return 1 

 

with get_cmdline_logger(verbose): 

ok = True 

for ok, message in check_config(config): 

if ok: 

print_ok(message) 

else: 

print_err(message) 

ok = False 

if ok: 

print_ok("+ all checks passed", fg="green") 

else: 

print_err("+ checks failed") 

return ok 

 

 

def init_db(reset, verbose, print_ok, print_err): 

"""creates empty tables in configured database, can be used to delete data from an existing 

database. 

""" 

 

with get_cmdline_logger(verbose): 

 

from datapool.config_handling import read_config 

config = read_config() 

 

if config is None: 

print_err("+ no config file found. please run 'pool init' first.") 

return 1 

 

already_exists = check_if_tables_exist(config.db) 

if already_exists: 

if reset: 

setup_fresh_db(config.db, verbose=verbose) 

else: 

print_err("+ database is already initialized, use --force TWICE to setup a fresh " 

"and empty database. YOU WILL LOOSE ALL EXISTING DATA !!!") 

return 1 

else: 

setup_db(config.db, verbose=verbose) 

 

print_ok("+ intialized db", fg="green") 

return 0 

 

 

def _setup_landing_zone(development_landing_zone, operational_landing_zone, is_first_time, reset, 

print_ok, print_err): 

 

print_ok("- setup development landing zone") 

if exists(development_landing_zone): 

if not reset: 

print_err(" - folder {} already exists.".format(development_landing_zone)) 

return 1 

else: 

try: 

shutil.rmtree(development_landing_zone) 

except Exception as e: 

print_err(" - could not delete folder {}".format(development_landing_zone)) 

print_err(" - error message is: {}".format(e.strerror)) 

return 1 

 

try: 

if is_first_time: 

print_ok("- operational landing zone is empty. create development landing zone with " 

"example files.") 

LandingZone.create_from_empty(development_landing_zone) 

else: 

print_ok("- copy files from operational landing zone.") 

LandingZone.create_from(development_landing_zone, operational_landing_zone) 

except IOError as e: 

print_err("- something went wrong: {}".format(e)) 

return 1 

return 0 

 

 

def _setup_db(development_landing_zone, config, is_first_time, reset, verbose, print_ok, print_err): 

 

from datapool.config_handling import config_for_develop_db 

from datapool.database import setup_db 

 

print_ok("- setup development data base") 

config_develop_db, __ = config_for_develop_db(development_landing_zone) 

conn_string = config_develop_db.connection_string 

try: 

tables_exist = check_if_tables_exist(config_develop_db) 

except InvalidOperationError: 

print_err(" - could not connect to {}. check database settings.".format(conn_string)) 

return 1 

 

if tables_exist and not reset: 

print_err(" - the database {} already holds datapool tables.".format(conn_string)) 

return 1 

 

if is_first_time: 

print_ok("- create tables and indices") 

try: 

setup_db(config_develop_db) 

except Exception as e: 

print_err(" - could not setup database {}".format(conn_string)) 

print_err(" - error message is: {}".format(e)) 

return 1 

else: 

print_ok("- clone meta data from operational db to development db") 

try: 

copy_db(config.db, config_develop_db, delete_existing=True, 

copy_signals=False, verbose=verbose) 

except Exception as e: 

print_err(" - cloning failed. error message is: {}".format(e)) 

return 1 

return 0 

 

 

def start_develop(development_landing_zone, reset, verbose, print_ok, print_err): 

"""creates local landing zone for integrating new devices, conversion scripts etc. 

either the configured operational landing zone is cloned or example files are written to 

the local landing zone. 

Further a local sqlite database is created. 

 

if "reset" is True existing folders and database will be overwritten. 

""" 

 

from datapool.config_handling import read_config 

 

config = read_config(DEVELOPZONE=development_landing_zone) 

operational_landing_zone = config.landing_zone.folder 

 

if not exists(operational_landing_zone): 

print_err("+ configured landing zone {} does not exist".format(operational_landing_zone)) 

return 1 

 

try: 

is_first_time = (os.listdir(operational_landing_zone) == []) 

except IOError as e: 

print_err("+ can not read {}: {}".format(operational_landing_zone, e)) 

return 1 

 

with get_cmdline_logger(verbose): 

exit_code = _setup_landing_zone(development_landing_zone, operational_landing_zone, 

is_first_time, reset, print_ok, print_err) 

if not exit_code: 

exit_code = _setup_db(development_landing_zone, config, is_first_time, reset, verbose, 

print_ok, print_err) 

 

if exit_code: 

print_err("+ setup failed") 

else: 

print_ok("+ setup done", fg="green") 

return exit_code 

 

 

def check_scripts(landing_zone, result_folder, verbose, print_ok, print_err): 

 

print_ok("- check landing zone at {}".format(landing_zone)) 

if not exists(landing_zone): 

print_err(" - folder {} does not exist".format(landing_zone)) 

return 1 

 

from datapool.data_conversion import check_conversion 

from datapool.config_handling import read_config 

config = read_config() 

 

if not result_folder: 

result_folder = tempfile.mkdtemp() 

else: 

if os.path.exists(result_folder): 

if not os.path.isdir(result_folder): 

print_err("+ given path {} exists but is not a folder".format(result_folder)) 

return 1 

else: 

os.makedirs(result_folder) 

print_ok("- created folder {}".format(result_folder)) 

 

with get_cmdline_logger(verbose): 

try: 

lz = LandingZone(landing_zone) 

except InvalidLandingZone as e: 

print_err(" - landing zone at {} invalid. reason: {}".format(landing_zone, e)) 

return 1 

found_errors = False 

for script, data_path in lz.conversion_scripts_and_data(): 

print_ok("- check {}".format(script)) 

for result in check_conversion(script, data_path, config, verbose): 

if isinstance(result, Exception): 

print_err(" - {}".format(result)) 

found_errors = True 

if not found_errors: 

# last result must be the tuple output_file, signals 

output_file, signals = result 

script_folder_name = os.path.basename(os.path.dirname(script)) 

csv_path = join(result_folder, script_folder_name) + ".csv" 

txt_path = join(result_folder, script_folder_name) + ".txt" 

 

csv_path, txt_path = enumerate_filename(csv_path, txt_path) 

shutil.copy(output_file, csv_path) 

print_ok("- wrote conversion result to {}".format(csv_path)) 

with open(txt_path, "w") as fh: 

print_signals(signals, file=fh) 

print_ok("- wrote conversion result to {}".format(txt_path)) 

 

if found_errors: 

print_err("+ checks failed. please fix this.") 

return 1 

print_ok("+ congratulations: checks succeeded.", fg="green") 

return 0 

 

 

def update_operational(development_landing_zone, verbose, overwrite, print_ok, print_err): 

 

from datapool.config_handling import read_config 

config = read_config() 

operational_landing_zone = config.landing_zone.folder 

 

lz = LandingZone(development_landing_zone) 

print_ok("- check before copying files around.") 

 

try: 

messages = lz.check_update_operational(operational_landing_zone) 

except InvalidLandingZone as e: 

print_err(" - {}".format(e)) 

return 1 

if messages: 

for message in messages: 

print_err(" - problem: {}".format(message)) 

if not overwrite: 

return 1 

else: 

print_ok(" - will ignore these and overwrite existing files") 

 

files = lz.update_operational(operational_landing_zone) 

for file in files: 

print_ok("- copied {}".format(file)) 

print_ok("+ copied {} files to {}".format(len(files), operational_landing_zone), fg="green") 

return 0 

 

 

@contextlib.contextmanager 

def _setup_test_db(landing_zone): 

from datapool.config_handling import config_for_develop_db 

from datapool.database import connect_to_db 

db_config, path = config_for_develop_db(landing_zone) 

 

shutil.copy(path, path + ".backup") 

engine = connect_to_db(db_config) 

 

yield engine 

 

engine.dispose() 

 

os.unlink(path) 

os.rename(path + ".backup", path) 

 

 

def check_yamls(landing_zone, verbose, print_ok, print_err): 

 

print_ok("- check yamls in landing zone at {}".format(landing_zone)) 

if not exists(landing_zone): 

print_err(" - folder {} does not exist".format(landing_zone)) 

return 1 

 

with get_cmdline_logger(verbose): 

try: 

lz = LandingZone(landing_zone) 

except InvalidLandingZone as e: 

print_err(" - landing zone at {} invalid. reason: {}".format(landing_zone, e)) 

return 1 

 

with _setup_test_db(landing_zone) as engine: 

found_errors = _check_lz(lz, engine, print_ok, print_err, verbose) 

if found_errors: 

print_err("+ all scripts checked") 

return 1 

print_ok("+ all scripts checked", fg="green") 

return 0 

 

 

def _check_lz(lz, engine, print_ok, print_err, verbose): 

found_errors = False 

for rel_path in lz.list_changed_files(): 

if not rel_path.endswith(".yaml"): 

continue 

print_ok("- check {}".format(join(lz.root_folder, rel_path))) 

for error in _check_yaml(engine, lz.root_folder, rel_path): 

found_errors = True 

print_err(" - got error: {}".format(error)) 

return found_errors 

 

 

def _check_yaml(engine, landing_zone_folder, rel_path): 

from datapool.landing_zone_structure import lookup_parser, lookup_checker 

parser = lookup_parser(rel_path) 

checker = lookup_checker(rel_path) 

if parser is None: 

yield Exception("no parser for {} implemented. maybe filename of misspelled " 

"or file is at wrong place".format(rel_path)) 

return 

if checker is None: 

full_path = join(landing_zone_folder, rel_path) 

raise RuntimeError("no checker for {} found, but parser exists".format(rel_path)) 

results = parser(landing_zone_folder, rel_path) 

for parse_result in results: 

if isinstance(parse_result, Exception): 

yield parse_result 

else: 

check_results = checker(parse_result, engine) 

for check_result in check_results: 

if isinstance(check_result, Exception): 

yield check_result