Coverage for src/configuraptor/core.py: 100%

166 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-22 14:32 +0200

1""" 

2Contains most of the loading logic. 

3""" 

4 

5import dataclasses as dc 

6import math 

7import types 

8import typing 

9import warnings 

10from collections import ChainMap 

11from pathlib import Path 

12 

13from typeguard import TypeCheckError 

14from typeguard import check_type as _check_type 

15 

16from . import loaders 

17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey 

18from .helpers import camel_to_snake 

19from .postpone import Postponed 

20 

21# T is a reusable typevar 

22T = typing.TypeVar("T") 

23# t_typelike is anything that can be type hinted 

24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union 

25# t_data is anything that can be fed to _load_data 

26T_data = str | Path | dict[str, typing.Any] 

27# c = a config class instance, can be any (user-defined) class 

28C = typing.TypeVar("C") 

29# type c is a config class 

30Type_C = typing.Type[C] 

31 

32 

33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]: 

34 """ 

35 If a key contains a dot, traverse the raw dict until the right key was found. 

36 

37 Example: 

38 key = some.nested.key 

39 raw = {"some": {"nested": {"key": {"with": "data"}}}} 

40 -> {"with": "data"} 

41 """ 

42 parts = key.split(".") 

43 while parts: 

44 raw = raw[parts.pop(0)] 

45 

46 return raw 

47 

48 

49def _guess_key(clsname: str) -> str: 

50 """ 

51 If no key is manually defined for `load_into`, \ 

52 the class' name is converted to snake_case to use as the default key. 

53 """ 

54 return camel_to_snake(clsname) 

55 

56 

57def __load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

58 """ 

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname. 

60 

61 E.g. class Tool will be mapped to key tool. 

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}} 

63 """ 

64 if isinstance(data, str): 

65 data = Path(data) 

66 if isinstance(data, Path): 

67 with data.open("rb") as f: 

68 loader = loaders.get(data.suffix) 

69 data = loader(f) 

70 

71 if not data: 

72 return {} 

73 

74 if key is None: 

75 # try to guess key by grabbing the first one or using the class name 

76 if len(data) == 1: 

77 key = list(data.keys())[0] 

78 elif classname is not None: 

79 key = _guess_key(classname) 

80 

81 if key: 

82 data = _data_for_nested_key(key, data) 

83 

84 if not data: 

85 raise ValueError("No data found!") 

86 

87 if not isinstance(data, dict): 

88 raise ValueError("Data is not a dict!") 

89 

90 return data 

91 

92 

93def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]: 

94 """ 

95 Wrapper around __load_data that retries with key="" if anything goes wrong. 

96 """ 

97 try: 

98 return __load_data(data, key, classname) 

99 except Exception as e: 

100 if key != "": 

101 return __load_data(data, "", classname) 

102 else: # pragma: no cover 

103 # key already was "", just return data! 

104 # (will probably not happen but fallback) 

105 return {} 

106 

107 

108def check_type(value: typing.Any, expected_type: T_typelike) -> bool: 

109 """ 

110 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.). 

111 

112 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError 

113 """ 

114 try: 

115 _check_type(value, expected_type) 

116 return True 

117 except TypeCheckError: 

118 return False 

119 

120 

121def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]: 

122 """ 

123 Make sure all values in 'data' are in line with the ones stored in 'annotations'. 

124 

125 If an annotated key in missing from data, it will be filled with None for convenience. 

126 

127 TODO: python 3.11 exception groups to throw multiple errors at once! 

128 """ 

129 # custom object to use instead of None, since typing.Optional can be None! 

130 # cast to T to make mypy happy 

131 notfound = typing.cast(T, object()) 

132 postponed = Postponed() 

133 

134 final: dict[str, T | None] = {} 

135 for key, _type in annotations.items(): 

136 compare = data.get(key, notfound) 

137 if compare is notfound: # pragma: nocover 

138 warnings.warn( 

139 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`" 

140 ) 

141 # skip! 

142 continue 

143 

144 if compare is postponed: 

145 # don't do anything with this item! 

146 continue 

147 

148 if not check_type(compare, _type): 

149 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type) 

150 

151 final[key] = compare 

152 

153 return final 

154 

155 

156def convert_config(items: dict[str, T]) -> dict[str, T]: 

157 """ 

158 Converts the config dict (from toml) or 'overwrites' dict in two ways. 

159 

160 1. removes any items where the value is None, since in that case the default should be used; 

161 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties. 

162 """ 

163 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None} 

164 

165 

166Type = typing.Type[typing.Any] 

167T_Type = typing.TypeVar("T_Type", bound=Type) 

168 

169 

170def is_builtin_type(_type: Type) -> bool: 

171 """ 

172 Returns whether _type is one of the builtin types. 

173 """ 

174 return _type.__module__ in ("__builtin__", "builtins") 

175 

176 

177# def is_builtin_class_instance(obj: typing.Any) -> bool: 

178# return is_builtin_type(obj.__class__) 

179 

180 

181def is_from_types_or_typing(_type: Type) -> bool: 

182 """ 

183 Returns whether _type is one of the stlib typing/types types. 

184 

185 e.g. types.UnionType or typing.Union 

186 """ 

187 return _type.__module__ in ("types", "typing") 

188 

189 

190def is_from_other_toml_supported_module(_type: Type) -> bool: 

191 """ 

192 Besides builtins, toml also supports 'datetime' and 'math' types, \ 

193 so this returns whether _type is a type from these stdlib modules. 

194 """ 

195 return _type.__module__ in ("datetime", "math") 

196 

197 

198def is_parameterized(_type: Type) -> bool: 

199 """ 

200 Returns whether _type is a parameterized type. 

201 

202 Examples: 

203 list[str] -> True 

204 str -> False 

205 """ 

206 return typing.get_origin(_type) is not None 

207 

208 

209def is_custom_class(_type: Type) -> bool: 

210 """ 

211 Tries to guess if _type is a builtin or a custom (user-defined) class. 

212 

213 Other logic in this module depends on knowing that. 

214 """ 

215 return ( 

216 type(_type) is type 

217 and not is_builtin_type(_type) 

218 and not is_from_other_toml_supported_module(_type) 

219 and not is_from_types_or_typing(_type) 

220 ) 

221 

222 

223def instance_of_custom_class(var: typing.Any) -> bool: 

224 """ 

225 Calls `is_custom_class` on an instance of a (possibly custom) class. 

226 """ 

227 return is_custom_class(var.__class__) 

228 

229 

230def is_optional(_type: Type | typing.Any) -> bool: 

231 """ 

232 Tries to guess if _type could be optional. 

233 

234 Examples: 

235 None -> True 

236 NoneType -> True 

237 typing.Union[str, None] -> True 

238 str | None -> True 

239 list[str | None] -> False 

240 list[str] -> False 

241 """ 

242 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan): 

243 # e.g. list[str] 

244 # will crash issubclass to test it first here 

245 return False 

246 

247 return ( 

248 _type is None 

249 or types.NoneType in typing.get_args(_type) # union with Nonetype 

250 or issubclass(types.NoneType, _type) 

251 or issubclass(types.NoneType, type(_type)) # no type # Nonetype 

252 ) 

253 

254 

255def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]: 

256 """ 

257 Get Field info for a dataclass cls. 

258 """ 

259 fields = getattr(cls, "__dataclass_fields__", {}) 

260 return fields.get(key) 

261 

262 

263def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]: 

264 """ 

265 For all annotations (recursively gathered from parents with `all_annotations`), \ 

266 try to resolve the tree of annotations. 

267 

268 Uses `load_into_recurse`, not itself directly. 

269 

270 Example: 

271 class First: 

272 key: str 

273 

274 class Second: 

275 other: First 

276 

277 # step 1 

278 cls = Second 

279 data = {"second": {"other": {"key": "anything"}}} 

280 annotations: {"other": First} 

281 

282 # step 1.5 

283 data = {"other": {"key": "anything"} 

284 annotations: {"other": First} 

285 

286 # step 2 

287 cls = First 

288 data = {"key": "anything"} 

289 annotations: {"key": str} 

290 

291 

292 TODO: python 3.11 exception groups to throw multiple errors at once! 

293 """ 

294 updated = {} 

295 

296 for _key, _type in annotations.items(): 

297 if _key in data: 

298 value: typing.Any = data[_key] # value can change so define it as any instead of T 

299 if is_parameterized(_type): 

300 origin = typing.get_origin(_type) 

301 arguments = typing.get_args(_type) 

302 if origin is list and arguments and is_custom_class(arguments[0]): 

303 subtype = arguments[0] 

304 value = [_load_into_recurse(subtype, subvalue) for subvalue in value] 

305 

306 elif origin is dict and arguments and is_custom_class(arguments[1]): 

307 # e.g. dict[str, Point] 

308 subkeytype, subvaluetype = arguments 

309 # subkey(type) is not a custom class, so don't try to convert it: 

310 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()} 

311 # elif origin is dict: 

312 # keep data the same 

313 elif origin is typing.Union and arguments: 

314 for arg in arguments: 

315 if is_custom_class(arg): 

316 value = _load_into_recurse(arg, value) 

317 else: 

318 # print(_type, arg, value) 

319 ... 

320 

321 # todo: other parameterized/unions/typing.Optional 

322 

323 elif is_custom_class(_type): 

324 # type must be C (custom class) at this point 

325 value = _load_into_recurse( 

326 # make mypy and pycharm happy by telling it _type is of type C... 

327 # actually just passing _type as first arg! 

328 typing.cast(Type_C[typing.Any], _type), 

329 value, 

330 ) 

331 

332 elif _key in cls.__dict__: 

333 # property has default, use that instead. 

334 value = cls.__dict__[_key] 

335 elif is_optional(_type): 

336 # type is optional and not found in __dict__ -> default is None 

337 value = None 

338 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING: 

339 # could have a default factory 

340 # todo: do something with field.default? 

341 value = field.default_factory() 

342 else: 

343 raise ConfigErrorMissingKey(_key, cls, _type) 

344 

345 updated[_key] = value 

346 

347 return updated 

348 

349 

350def _all_annotations(cls: Type) -> ChainMap[str, Type]: 

351 """ 

352 Returns a dictionary-like ChainMap that includes annotations for all \ 

353 attributes defined in cls or inherited from superclasses. 

354 """ 

355 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__)) 

356 

357 

358def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]: 

359 """ 

360 Wrapper around `_all_annotations` that filters away any keys in _except. 

361 

362 It also flattens the ChainMap to a regular dict. 

363 """ 

364 if _except is None: 

365 _except = set() 

366 

367 _all = _all_annotations(cls) 

368 return {k: v for k, v in _all.items() if k not in _except} 

369 

370 

371def check_and_convert_data( 

372 cls: typing.Type[C], 

373 data: dict[str, typing.Any], 

374 _except: typing.Iterable[str], 

375 strict: bool = True, 

376) -> dict[str, typing.Any]: 

377 """ 

378 Based on class annotations, this prepares the data for `load_into_recurse`. 

379 

380 1. convert config-keys to python compatible config_keys 

381 2. loads custom class type annotations with the same logic (see also `load_recursive`) 

382 3. ensures the annotated types match the actual types after loading the config file. 

383 """ 

384 annotations = all_annotations(cls, _except=_except) 

385 

386 to_load = convert_config(data) 

387 to_load = load_recursive(cls, to_load, annotations) 

388 if strict: 

389 to_load = ensure_types(to_load, annotations) 

390 

391 return to_load 

392 

393 

394def _load_into_recurse( 

395 cls: typing.Type[C], 

396 data: dict[str, typing.Any], 

397 init: dict[str, typing.Any] = None, 

398 strict: bool = True, 

399) -> C: 

400 """ 

401 Loads an instance of `cls` filled with `data`. 

402 

403 Uses `load_recursive` to load any fillable annotated properties (see that method for an example). 

404 `init` can be used to optionally pass extra __init__ arguments. \ 

405 NOTE: This will overwrite a config key with the same name! 

406 """ 

407 if init is None: 

408 init = {} 

409 

410 # fixme: cls.__init__ can set other keys than the name is in kwargs!! 

411 

412 if dc.is_dataclass(cls): 

413 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict) 

414 to_load |= init # add extra init variables (should not happen for a dataclass but whatev) 

415 

416 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`) 

417 inst = typing.cast(C, cls(**to_load)) 

418 else: 

419 inst = cls(**init) 

420 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict) 

421 inst.__dict__.update(**to_load) 

422 

423 return inst 

424 

425 

426def _load_into_instance( 

427 inst: C, 

428 cls: typing.Type[C], 

429 data: dict[str, typing.Any], 

430 init: dict[str, typing.Any] = None, 

431 strict: bool = True, 

432) -> C: 

433 """ 

434 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \ 

435 and thus does not support init. 

436 

437 """ 

438 if init is not None: 

439 raise ValueError("Can not init an existing instance!") 

440 

441 existing_data = inst.__dict__ 

442 

443 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict) 

444 

445 inst.__dict__.update(**to_load) 

446 

447 return inst 

448 

449 

450def load_into_class( 

451 cls: typing.Type[C], 

452 data: T_data, 

453 /, 

454 key: str = None, 

455 init: dict[str, typing.Any] = None, 

456 strict: bool = True, 

457) -> C: 

458 """ 

459 Shortcut for _load_data + load_into_recurse. 

460 """ 

461 to_load = _load_data(data, key, cls.__name__) 

462 return _load_into_recurse(cls, to_load, init=init, strict=strict) 

463 

464 

465def load_into_instance( 

466 inst: C, 

467 data: T_data, 

468 /, 

469 key: str = None, 

470 init: dict[str, typing.Any] = None, 

471 strict: bool = True, 

472) -> C: 

473 """ 

474 Shortcut for _load_data + load_into_existing. 

475 """ 

476 cls = inst.__class__ 

477 to_load = _load_data(data, key, cls.__name__) 

478 return _load_into_instance(inst, cls, to_load, init=init, strict=strict) 

479 

480 

481def load_into( 

482 cls: typing.Type[C], 

483 data: T_data, 

484 /, 

485 key: str = None, 

486 init: dict[str, typing.Any] = None, 

487 strict: bool = True, 

488) -> C: 

489 """ 

490 Load your config into a class (instance). 

491 

492 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only 

493 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`. 

494 

495 Args: 

496 cls: either a class or an existing instance of that class. 

497 data: can be a dictionary or a path to a file to load (as pathlib.Path or str) 

498 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific') 

499 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already) 

500 strict: enable type checks or allow anything? 

501 

502 """ 

503 if not isinstance(cls, type): 

504 # would not be supported according to mypy, but you can still load_into(instance) 

505 return load_into_instance(cls, data, key=key, init=init, strict=strict) 

506 

507 # make mypy and pycharm happy by telling it cls is of type C and not just 'type' 

508 # _cls = typing.cast(typing.Type[C], cls) 

509 return load_into_class(cls, data, key=key, init=init, strict=strict)