Coverage for src/configuraptor/core.py: 100%
166 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-22 14:32 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-22 14:32 +0200
1"""
2Contains most of the loading logic.
3"""
5import dataclasses as dc
6import math
7import types
8import typing
9import warnings
10from collections import ChainMap
11from pathlib import Path
13from typeguard import TypeCheckError
14from typeguard import check_type as _check_type
16from . import loaders
17from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey
18from .helpers import camel_to_snake
19from .postpone import Postponed
21# T is a reusable typevar
22T = typing.TypeVar("T")
23# t_typelike is anything that can be type hinted
24T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union
25# t_data is anything that can be fed to _load_data
26T_data = str | Path | dict[str, typing.Any]
27# c = a config class instance, can be any (user-defined) class
28C = typing.TypeVar("C")
29# type c is a config class
30Type_C = typing.Type[C]
33def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:
34 """
35 If a key contains a dot, traverse the raw dict until the right key was found.
37 Example:
38 key = some.nested.key
39 raw = {"some": {"nested": {"key": {"with": "data"}}}}
40 -> {"with": "data"}
41 """
42 parts = key.split(".")
43 while parts:
44 raw = raw[parts.pop(0)]
46 return raw
49def _guess_key(clsname: str) -> str:
50 """
51 If no key is manually defined for `load_into`, \
52 the class' name is converted to snake_case to use as the default key.
53 """
54 return camel_to_snake(clsname)
57def __load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:
58 """
59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.
61 E.g. class Tool will be mapped to key tool.
62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}
63 """
64 if isinstance(data, str):
65 data = Path(data)
66 if isinstance(data, Path):
67 with data.open("rb") as f:
68 loader = loaders.get(data.suffix)
69 data = loader(f)
71 if not data:
72 return {}
74 if key is None:
75 # try to guess key by grabbing the first one or using the class name
76 if len(data) == 1:
77 key = list(data.keys())[0]
78 elif classname is not None:
79 key = _guess_key(classname)
81 if key:
82 data = _data_for_nested_key(key, data)
84 if not data:
85 raise ValueError("No data found!")
87 if not isinstance(data, dict):
88 raise ValueError("Data is not a dict!")
90 return data
93def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:
94 """
95 Wrapper around __load_data that retries with key="" if anything goes wrong.
96 """
97 try:
98 return __load_data(data, key, classname)
99 except Exception as e:
100 if key != "":
101 return __load_data(data, "", classname)
102 else: # pragma: no cover
103 # key already was "", just return data!
104 # (will probably not happen but fallback)
105 return {}
108def check_type(value: typing.Any, expected_type: T_typelike) -> bool:
109 """
110 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).
112 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError
113 """
114 try:
115 _check_type(value, expected_type)
116 return True
117 except TypeCheckError:
118 return False
121def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:
122 """
123 Make sure all values in 'data' are in line with the ones stored in 'annotations'.
125 If an annotated key in missing from data, it will be filled with None for convenience.
127 TODO: python 3.11 exception groups to throw multiple errors at once!
128 """
129 # custom object to use instead of None, since typing.Optional can be None!
130 # cast to T to make mypy happy
131 notfound = typing.cast(T, object())
132 postponed = Postponed()
134 final: dict[str, T | None] = {}
135 for key, _type in annotations.items():
136 compare = data.get(key, notfound)
137 if compare is notfound: # pragma: nocover
138 warnings.warn(
139 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"
140 )
141 # skip!
142 continue
144 if compare is postponed:
145 # don't do anything with this item!
146 continue
148 if not check_type(compare, _type):
149 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)
151 final[key] = compare
153 return final
156def convert_config(items: dict[str, T]) -> dict[str, T]:
157 """
158 Converts the config dict (from toml) or 'overwrites' dict in two ways.
160 1. removes any items where the value is None, since in that case the default should be used;
161 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.
162 """
163 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}
166Type = typing.Type[typing.Any]
167T_Type = typing.TypeVar("T_Type", bound=Type)
170def is_builtin_type(_type: Type) -> bool:
171 """
172 Returns whether _type is one of the builtin types.
173 """
174 return _type.__module__ in ("__builtin__", "builtins")
177# def is_builtin_class_instance(obj: typing.Any) -> bool:
178# return is_builtin_type(obj.__class__)
181def is_from_types_or_typing(_type: Type) -> bool:
182 """
183 Returns whether _type is one of the stlib typing/types types.
185 e.g. types.UnionType or typing.Union
186 """
187 return _type.__module__ in ("types", "typing")
190def is_from_other_toml_supported_module(_type: Type) -> bool:
191 """
192 Besides builtins, toml also supports 'datetime' and 'math' types, \
193 so this returns whether _type is a type from these stdlib modules.
194 """
195 return _type.__module__ in ("datetime", "math")
198def is_parameterized(_type: Type) -> bool:
199 """
200 Returns whether _type is a parameterized type.
202 Examples:
203 list[str] -> True
204 str -> False
205 """
206 return typing.get_origin(_type) is not None
209def is_custom_class(_type: Type) -> bool:
210 """
211 Tries to guess if _type is a builtin or a custom (user-defined) class.
213 Other logic in this module depends on knowing that.
214 """
215 return (
216 type(_type) is type
217 and not is_builtin_type(_type)
218 and not is_from_other_toml_supported_module(_type)
219 and not is_from_types_or_typing(_type)
220 )
223def instance_of_custom_class(var: typing.Any) -> bool:
224 """
225 Calls `is_custom_class` on an instance of a (possibly custom) class.
226 """
227 return is_custom_class(var.__class__)
230def is_optional(_type: Type | typing.Any) -> bool:
231 """
232 Tries to guess if _type could be optional.
234 Examples:
235 None -> True
236 NoneType -> True
237 typing.Union[str, None] -> True
238 str | None -> True
239 list[str | None] -> False
240 list[str] -> False
241 """
242 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):
243 # e.g. list[str]
244 # will crash issubclass to test it first here
245 return False
247 return (
248 _type is None
249 or types.NoneType in typing.get_args(_type) # union with Nonetype
250 or issubclass(types.NoneType, _type)
251 or issubclass(types.NoneType, type(_type)) # no type # Nonetype
252 )
255def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:
256 """
257 Get Field info for a dataclass cls.
258 """
259 fields = getattr(cls, "__dataclass_fields__", {})
260 return fields.get(key)
263def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:
264 """
265 For all annotations (recursively gathered from parents with `all_annotations`), \
266 try to resolve the tree of annotations.
268 Uses `load_into_recurse`, not itself directly.
270 Example:
271 class First:
272 key: str
274 class Second:
275 other: First
277 # step 1
278 cls = Second
279 data = {"second": {"other": {"key": "anything"}}}
280 annotations: {"other": First}
282 # step 1.5
283 data = {"other": {"key": "anything"}
284 annotations: {"other": First}
286 # step 2
287 cls = First
288 data = {"key": "anything"}
289 annotations: {"key": str}
292 TODO: python 3.11 exception groups to throw multiple errors at once!
293 """
294 updated = {}
296 for _key, _type in annotations.items():
297 if _key in data:
298 value: typing.Any = data[_key] # value can change so define it as any instead of T
299 if is_parameterized(_type):
300 origin = typing.get_origin(_type)
301 arguments = typing.get_args(_type)
302 if origin is list and arguments and is_custom_class(arguments[0]):
303 subtype = arguments[0]
304 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]
306 elif origin is dict and arguments and is_custom_class(arguments[1]):
307 # e.g. dict[str, Point]
308 subkeytype, subvaluetype = arguments
309 # subkey(type) is not a custom class, so don't try to convert it:
310 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}
311 # elif origin is dict:
312 # keep data the same
313 elif origin is typing.Union and arguments:
314 for arg in arguments:
315 if is_custom_class(arg):
316 value = _load_into_recurse(arg, value)
317 else:
318 # print(_type, arg, value)
319 ...
321 # todo: other parameterized/unions/typing.Optional
323 elif is_custom_class(_type):
324 # type must be C (custom class) at this point
325 value = _load_into_recurse(
326 # make mypy and pycharm happy by telling it _type is of type C...
327 # actually just passing _type as first arg!
328 typing.cast(Type_C[typing.Any], _type),
329 value,
330 )
332 elif _key in cls.__dict__:
333 # property has default, use that instead.
334 value = cls.__dict__[_key]
335 elif is_optional(_type):
336 # type is optional and not found in __dict__ -> default is None
337 value = None
338 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:
339 # could have a default factory
340 # todo: do something with field.default?
341 value = field.default_factory()
342 else:
343 raise ConfigErrorMissingKey(_key, cls, _type)
345 updated[_key] = value
347 return updated
350def _all_annotations(cls: Type) -> ChainMap[str, Type]:
351 """
352 Returns a dictionary-like ChainMap that includes annotations for all \
353 attributes defined in cls or inherited from superclasses.
354 """
355 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))
358def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:
359 """
360 Wrapper around `_all_annotations` that filters away any keys in _except.
362 It also flattens the ChainMap to a regular dict.
363 """
364 if _except is None:
365 _except = set()
367 _all = _all_annotations(cls)
368 return {k: v for k, v in _all.items() if k not in _except}
371def check_and_convert_data(
372 cls: typing.Type[C],
373 data: dict[str, typing.Any],
374 _except: typing.Iterable[str],
375 strict: bool = True,
376) -> dict[str, typing.Any]:
377 """
378 Based on class annotations, this prepares the data for `load_into_recurse`.
380 1. convert config-keys to python compatible config_keys
381 2. loads custom class type annotations with the same logic (see also `load_recursive`)
382 3. ensures the annotated types match the actual types after loading the config file.
383 """
384 annotations = all_annotations(cls, _except=_except)
386 to_load = convert_config(data)
387 to_load = load_recursive(cls, to_load, annotations)
388 if strict:
389 to_load = ensure_types(to_load, annotations)
391 return to_load
394def _load_into_recurse(
395 cls: typing.Type[C],
396 data: dict[str, typing.Any],
397 init: dict[str, typing.Any] = None,
398 strict: bool = True,
399) -> C:
400 """
401 Loads an instance of `cls` filled with `data`.
403 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).
404 `init` can be used to optionally pass extra __init__ arguments. \
405 NOTE: This will overwrite a config key with the same name!
406 """
407 if init is None:
408 init = {}
410 # fixme: cls.__init__ can set other keys than the name is in kwargs!!
412 if dc.is_dataclass(cls):
413 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict)
414 to_load |= init # add extra init variables (should not happen for a dataclass but whatev)
416 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)
417 inst = typing.cast(C, cls(**to_load))
418 else:
419 inst = cls(**init)
420 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict)
421 inst.__dict__.update(**to_load)
423 return inst
426def _load_into_instance(
427 inst: C,
428 cls: typing.Type[C],
429 data: dict[str, typing.Any],
430 init: dict[str, typing.Any] = None,
431 strict: bool = True,
432) -> C:
433 """
434 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \
435 and thus does not support init.
437 """
438 if init is not None:
439 raise ValueError("Can not init an existing instance!")
441 existing_data = inst.__dict__
443 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict)
445 inst.__dict__.update(**to_load)
447 return inst
450def load_into_class(
451 cls: typing.Type[C],
452 data: T_data,
453 /,
454 key: str = None,
455 init: dict[str, typing.Any] = None,
456 strict: bool = True,
457) -> C:
458 """
459 Shortcut for _load_data + load_into_recurse.
460 """
461 to_load = _load_data(data, key, cls.__name__)
462 return _load_into_recurse(cls, to_load, init=init, strict=strict)
465def load_into_instance(
466 inst: C,
467 data: T_data,
468 /,
469 key: str = None,
470 init: dict[str, typing.Any] = None,
471 strict: bool = True,
472) -> C:
473 """
474 Shortcut for _load_data + load_into_existing.
475 """
476 cls = inst.__class__
477 to_load = _load_data(data, key, cls.__name__)
478 return _load_into_instance(inst, cls, to_load, init=init, strict=strict)
481def load_into(
482 cls: typing.Type[C],
483 data: T_data,
484 /,
485 key: str = None,
486 init: dict[str, typing.Any] = None,
487 strict: bool = True,
488) -> C:
489 """
490 Load your config into a class (instance).
492 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only
493 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.
495 Args:
496 cls: either a class or an existing instance of that class.
497 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)
498 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')
499 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)
500 strict: enable type checks or allow anything?
502 """
503 if not isinstance(cls, type):
504 # would not be supported according to mypy, but you can still load_into(instance)
505 return load_into_instance(cls, data, key=key, init=init, strict=strict)
507 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'
508 # _cls = typing.cast(typing.Type[C], cls)
509 return load_into_class(cls, data, key=key, init=init, strict=strict)