Create a system to map between info.json and config.h/rules.mk (#11548)

* generate rules.mk from a json mapping

* generate rules.mk from a json mapping

* support for config.h from json maps

* improve the mapping system

* document the mapping system

* move data/maps to data/mappings

* fix flake8 errors

* fixup LED_MATRIX_DRIVER

* remove product and description from the vision_division keymap level

* reduce the complexity of generate-rules-mk

* add tests for the generate commands

* fix qmk doctor when submodules are not clean

bin/qmk

@@ -27,7 +27,8 @@ def _check_modules(requirements):
                 line = line.split('#')[0]
 
             module = dict()
-            module['name'] = module['import'] = line.split('=')[0] if '=' in line else line
+            module['name'] = line.split('=')[0] if '=' in line else line
+            module['import'] = module['name'].replace('-', '_')
 
             # Not every module is importable by its own name.
             if module['name'] == "pep8-naming":

data/mappings/info_config.json

@@ -0,0 +1,42 @@
+# This file maps keys between `config.h` and `info.json`. It is used by QMK 
+# to correctly and consistently map back and forth between the two systems.
+{
+    # Format:
+    # <config.h key>: {"info_key": <info.json key>, ["value_type": <value_type>], ["to_json": <true/false>], ["to_c": <true/false>]}
+    # value_type: one of "array", "array.int", "int", "hex", "list", "mapping"
+    # to_json: Default `true`. Set to `false` to exclude this mapping from info.json
+    # to_c: Default `true`. Set to `false` to exclude this mapping from config.h
+    # warn_duplicate: Default `true`. Set to `false` to turn off warning when a value exists in both places
+    "DEBOUNCE": {"info_key": "debounce", "value_type": "int"}
+    "DEVICE_VER": {"info_key": "usb.device_ver", "value_type": "hex"},
+    "DESCRIPTION": {"info_key": "keyboard_folder", "to_json": false},
+    "DIODE_DIRECTION": {"info_key": "diode_direction"},
+    "LAYOUTS": {"info_key": "layout_aliases", "value_type": "mapping"},
+    "LED_CAPS_LOCK_PIN": {"info_key": "indicators.caps_lock"},
+    "LED_NUM_LOCK_PIN": {"info_key": "indicators.num_lock"},
+    "LED_SCROLL_LOCK_PIN": {"info_key": "indicators.scroll_lock"},
+    "MANUFACTURER": {"info_key": "manufacturer"},
+    "RGB_DI_PIN": {"info_key": "rgblight.pin"},
+    "RGBLED_NUM": {"info_key": "rgblight.led_count", "value_type": "int"},
+    "RGBLED_SPLIT": {"info_key": "rgblight.split_count", "value_type": "array.int"},
+    "RGBLIGHT_ANIMATIONS": {"info_key": "rgblight.animations.all", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_ALTERNATING": {"info_key": "rgblight.animations.alternating", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_BREATHING": {"info_key": "rgblight.animations.breathing", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_CHRISTMAS": {"info_key": "rgblight.animations.christmas", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_KNIGHT": {"info_key": "rgblight.animations.knight", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_RAINBOW_MOOD": {"info_key": "rgblight.animations.rainbow_mood", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_RAINBOW_SWIRL": {"info_key": "rgblight.animations.rainbow_swirl", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_RGB_TEST": {"info_key": "rgblight.animations.rgb_test", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_SNAKE": {"info_key": "rgblight.animations.snake", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_STATIC_GRADIENT": {"info_key": "rgblight.animations.static_gradient", "value_type": "bool"},
+    "RGBLIGHT_EFFECT_TWINKLE": {"info_key": "rgblight.animations.twinkle"},
+    "RGBLIGHT_LIMIT_VAL": {"info_key": "rgblight.max_brightness", "value_type": "int"},
+    "RGBLIGHT_HUE_STEP": {"info_key": "rgblight.hue_steps", "value_type": "int"},
+    "RGBLIGHT_SAT_STEP": {"info_key": "rgblight.saturation_steps", "value_type": "int"},
+    "RGBLIGHT_VAL_STEP": {"info_key": "rgblight.brightness_steps", "value_type": "int"},
+    "RGBLIGHT_SLEEP": {"info_key": "rgblight.sleep", "value_type": "bool"},
+    "RGBLIGHT_SPLIT": {"info_key": "rgblight.split", "value_type": "bool"},
+    "PRODUCT": {"info_key": "keyboard_folder", "to_json": false},
+    "PRODUCT_ID": {"info_key": "usb.pid", "value_type": "hex"},
+    "VENDOR_ID": {"info_key": "usb.vid", "value_type": "hex"}
+}

data/mappings/info_rules.json

@@ -0,0 +1,15 @@
+# This file maps keys between `rules.mk` and `info.json`. It is used by QMK 
+# to correctly and consistently map back and forth between the two systems.
+{
+    # Format:
+    # <rules.mk key>: {"info_key": <info.json key>, ["value_type": <value_type>], ["to_json": <true/false>], ["to_c": <true/false>]}
+    # value_type: one of "array", "array.int", "int", "list", "hex", "mapping"
+    # to_json: Default `true`. Set to `false` to exclude this mapping from info.json
+    # to_c: Default `true`. Set to `false` to exclude this mapping from rules.mk
+    # warn_duplicate: Default `true`. Set to `false` to turn off warning when a value exists in both places
+    "BOARD": {"info_key": "board"},
+    "BOOTLOADER": {"info_key": "bootloader", "warn_duplicate": false},
+    "LAYOUTS": {"info_key": "community_layouts", "value_type": "list"},
+    "LED_MATRIX_DRIVER": {"info_key": "led_matrix.driver"},
+    "MCU": {"info_key": "processor", "warn_duplicate": false},
+}

docs/data_driven_config.md

@@ -12,17 +12,18 @@ Now we have support for generating `rules.mk` and `config.h` values from `info.j
 
 ## Overview
 
-On the C side of things nothing really changes. When you need to create a new rule or define you follow the same process:
+On the C side of things nothing changes. When you need to create a new rule or define you follow the same process:
 
 1. Add it to `docs/config_options.md`
 1. Set a default in the appropriate core file
-1. Add your `ifdef` and/or `#ifdef` statements as needed
+1. Add your ifdef statements as needed
 
 You will then need to add support for your new configuration to `info.json`. The basic process is:
 
 1. Add it to the schema in `data/schemas/keyboards.jsonschema`
-1. Add code to extract it from `config.h`/`rules.mk` to `lib/python/qmk/info.py`
-1. Add code to generate it to one of:
+1. Add a mapping in `data/maps`
+1. (optional and discoraged) Add code to extract/generate it to:
+  * `lib/python/qmk/info.py`
   * `lib/python/qmk/cli/generate/config_h.py`
   * `lib/python/qmk/cli/generate/rules_mk.py`
 
@@ -32,12 +33,43 @@ This section describes adding support for a `config.h`/`rules.mk` value to info.
 
 ### Add it to the schema
 
-QMK maintains schema files in `data/schemas`. The values that go into keyboard-specific `info.json` files are kept in `keyboard.jsonschema`. Any value you want to make available to end users to edit must go in here.
+QMK maintains [jsonschema](https://json-schema.org/) files in `data/schemas`. The values that go into keyboard-specific `info.json` files are kept in `keyboard.jsonschema`. Any value you want to make available to end users to edit must go in here.
 
-In some cases you can simply add a new top-level key. Some examples to follow are `keyboard_name`, `maintainer`, `processor`, and `url`. This is appropriate when your option is self-contained and not directly related to other options. In other cases you should group like options together in an `object`. This is particularly true when adding support for a feature. Some examples to follow for this are `indicators`, `matrix_pins`, and `rgblight`. If you are not sure how to integrate your new option(s) [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and start a conversation there.
+In some cases you can simply add a new top-level key. Some examples to follow are `keyboard_name`, `maintainer`, `processor`, and `url`. This is appropriate when your option is self-contained and not directly related to other options.
+
+In other cases you should group like options together in an `object`. This is particularly true when adding support for a feature. Some examples to follow for this are `indicators`, `matrix_pins`, and `rgblight`. If you are not sure how to integrate your new option(s) [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and start a conversation there.
+
+### Add a mapping
+
+In most cases you can add a simple mapping. These are maintained as JSON files in `data/mappings/info_config.json` and `data/mappings/info_rules.json`, and control mapping for `config.h` and `rules.mk`, respectively. Each mapping is keyed by the `config.h` or `rules.mk` variable, and the value is a hash with the following keys:
+
+* `info_key`: (required) The location within `info.json` for this value. See below.
+* `value_type`: (optional) Default `str`. The format for this variable's value. See below.
+* `to_json`: (optional) Default `true`. Set to `false` to exclude this mapping from info.json
+* `to_c`: (optional) Default `true`. Set to `false` to exclude this mapping from config.h
+* `warn_duplicate`: (optional) Default `true`. Set to `false` to turn off warning when a value exists in both places
+
+#### Info Key
+
+We use JSON dot notation to address variables within info.json. For example, to access `info_json["rgblight"]["split_count"]` I would specify `rgblight.split_count`. This allows you to address deeply nested keys with a simple string.
+
+Under the hood we use [Dotty Dict](https://dotty-dict.readthedocs.io/en/latest/), you can refer to that documentation for how these strings are converted to object access.
+
+#### Value Types
+
+By default we treat all values as simple strings. If your value is more complex you can use one of these types to intelligently parse the data:
+
+* `array`: A comma separated array of strings
+* `array.int`: A comma separated array of integers
+* `int`: An integer
+* `hex`: A number formatted as hex
+* `list`: A space separate array of strings
+* `mapping`: A hash of key/value pairs
 
 ### Add code to extract it
 
+Most use cases can be solved by the mapping files described above. If yours can't you can instead write code to extract your config values.
+
 Whenever QMK generates a complete `info.json` it extracts information from `config.h` and `rules.mk`. You will need to add code for your new config value to `lib/python/qmk/info.py`. Typically this means adding a new `_extract_<feature>()` function and then calling your function in either `_extract_config_h()` or `_extract_rules_mk()`.
 
 If you are not sure how to edit this file or are not comfortable with Python [open an issue](https://github.com/qmk/qmk_firmware/issues/new?assignees=&labels=cli%2C+python&template=other_issues.md&title=) or [join #cli on Discord](https://discord.gg/heQPAgy) and someone can help you with this part.

keyboards/handwired/pytest/basic/info.json

@@ -0,0 +1,10 @@
+{
+  "maintainer": "qmk",
+  "layouts": {
+    "LAYOUT_custom": {
+      "layout": [
+        { "label": "KC_Q", "matrix": [0, 0], "w": 1, "x": 0, "y": 0 }
+      ]
+    }
+  }
+}

keyboards/vision_division/keymaps/default/config.h

@@ -8,7 +8,6 @@
 #define VENDOR_ID       0xFEED
 #define DEVICE_VER      0x0001
 #define MANUFACTURER    IBNobody
-#define PRODUCT         Vision Division
 
 #define MATRIX_ROWS     6
 #define MATRIX_ROW_PINS { C2, C3, F4, F5, F6, F7 }

lib/python/qmk/cli/doctor.py

@@ -107,9 +107,9 @@ def doctor(cli):
             submodules.update()
             sub_ok = check_submodules()
 
-        if CheckStatus.ERROR in sub_ok:
+        if sub_ok == CheckStatus.ERROR:
             status = CheckStatus.ERROR
-        elif CheckStatus.WARNING in sub_ok and status == CheckStatus.OK:
+        elif sub_ok == CheckStatus.WARNING and status == CheckStatus.OK:
             status = CheckStatus.WARNING
 
     # Report a summary of our findings to the user

lib/python/qmk/cli/generate/config_h.py

@@ -1,62 +1,14 @@
 """Used by the make system to generate info_config.h from info.json.
 """
+from pathlib import Path
+
+from dotty_dict import dotty
 from milc import cli
 
-from qmk.constants import LED_INDICATORS
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.info import info_json, rgblight_animations, rgblight_properties, rgblight_toggles
+from qmk.info import _json_load, info_json
 from qmk.path import is_keyboard, normpath
 
-usb_prop_map = {
-    'vid': 'VENDOR_ID',
-    'pid': 'PRODUCT_ID',
-    'device_ver': 'DEVICE_VER',
-}
-
-
-def debounce(debounce):
-    """Return the config.h lines that set debounce
-    """
-    return """
-#ifndef DEBOUNCE
-#   define DEBOUNCE %s
-#endif // DEBOUNCE
-""" % debounce
-
-
-def diode_direction(diode_direction):
-    """Return the config.h lines that set diode direction
-    """
-    return """
-#ifndef DIODE_DIRECTION
-#   define DIODE_DIRECTION %s
-#endif // DIODE_DIRECTION
-""" % diode_direction
-
-
-def keyboard_name(keyboard_name):
-    """Return the config.h lines that set the keyboard's name.
-    """
-    return """
-#ifndef DESCRIPTION
-#    define DESCRIPTION %s
-#endif // DESCRIPTION
-
-#ifndef PRODUCT
-#    define PRODUCT %s
-#endif // PRODUCT
-""" % (keyboard_name.replace("'", ""), keyboard_name.replace("'", ""))
-
-
-def manufacturer(manufacturer):
-    """Return the config.h lines that set the manufacturer.
-    """
-    return """
-#ifndef MANUFACTURER
-#    define MANUFACTURER %s
-#endif // MANUFACTURER
-""" % (manufacturer.replace("'", ""))
-
 
 def direct_pins(direct_pins):
     """Return the config.h lines that set the direct pins.
@@ -72,80 +24,34 @@ def direct_pins(direct_pins):
 
     return """
 #ifndef MATRIX_COLS
-#    define MATRIX_COLS %s
+#   define MATRIX_COLS %s
 #endif // MATRIX_COLS
 
 #ifndef MATRIX_ROWS
-#    define MATRIX_ROWS %s
+#   define MATRIX_ROWS %s
 #endif // MATRIX_ROWS
 
 #ifndef DIRECT_PINS
-#    define DIRECT_PINS {%s}
+#   define DIRECT_PINS {%s}
 #endif // DIRECT_PINS
 """ % (col_count, row_count, ','.join(rows))
 
 
-def col_pins(col_pins):
-    """Return the config.h lines that set the column pins.
-    """
-    cols = ','.join(map(str, [pin or 'NO_PIN' for pin in col_pins]))
-    col_num = len(col_pins)
-
-    return """
-#ifndef MATRIX_COLS
-#    define MATRIX_COLS %s
-#endif // MATRIX_COLS
-
-#ifndef MATRIX_COL_PINS
-#    define MATRIX_COL_PINS {%s}
-#endif // MATRIX_COL_PINS
-""" % (col_num, cols)
-
-
-def row_pins(row_pins):
-    """Return the config.h lines that set the row pins.
-    """
-    rows = ','.join(map(str, [pin or 'NO_PIN' for pin in row_pins]))
-    row_num = len(row_pins)
-
-    return """
-#ifndef MATRIX_ROWS
-#    define MATRIX_ROWS %s
-#endif // MATRIX_ROWS
-
-#ifndef MATRIX_ROW_PINS
-#    define MATRIX_ROW_PINS {%s}
-#endif // MATRIX_ROW_PINS
-""" % (row_num, rows)
-
-
-def indicators(config):
-    """Return the config.h lines that setup LED indicators.
+def pin_array(define, pins):
+    """Return the config.h lines that set a pin array.
     """
-    defines = []
+    pin_num = len(pins)
+    pin_array = ', '.join(map(str, [pin or 'NO_PIN' for pin in pins]))
 
-    for led, define in LED_INDICATORS.items():
-        if led in config:
-            defines.append('')
-            defines.append('#ifndef %s' % (define,))
-            defines.append('#   define %s %s' % (define, config[led]))
-            defines.append('#endif // %s' % (define,))
+    return f"""
+#ifndef {define}S
+#   define {define}S {pin_num}
+#endif // {define}S
 
-    return '\n'.join(defines)
-
-
-def layout_aliases(layout_aliases):
-    """Return the config.h lines that setup layout aliases.
-    """
-    defines = []
-
-    for alias, layout in layout_aliases.items():
-        defines.append('')
-        defines.append('#ifndef %s' % (alias,))
-        defines.append('#   define %s %s' % (alias, layout))
-        defines.append('#endif // %s' % (alias,))
-
-    return '\n'.join(defines)
+#ifndef {define}_PINS
+#   define {define}_PINS {{ {pin_array} }}
+#endif // {define}_PINS
+"""
 
 
 def matrix_pins(matrix_pins):
@@ -157,58 +63,14 @@ def matrix_pins(matrix_pins):
         pins.append(direct_pins(matrix_pins['direct']))
 
     if 'cols' in matrix_pins:
-        pins.append(col_pins(matrix_pins['cols']))
+        pins.append(pin_array('MATRIX_COL', matrix_pins['cols']))
 
     if 'rows' in matrix_pins:
-        pins.append(row_pins(matrix_pins['rows']))
+        pins.append(pin_array('MATRIX_ROW', matrix_pins['rows']))
 
     return '\n'.join(pins)
 
 
-def rgblight(config):
-    """Return the config.h lines that setup rgblight.
-    """
-    rgblight_config = []
-
-    for json_key, config_key in rgblight_properties.items():
-        if json_key in config:
-            rgblight_config.append('')
-            rgblight_config.append('#ifndef %s' % (config_key[0],))
-            rgblight_config.append('#   define %s %s' % (config_key[0], config[json_key]))
-            rgblight_config.append('#endif // %s' % (config_key[0],))
-
-    for json_key, config_key in rgblight_toggles.items():
-        if config.get(json_key):
-            rgblight_config.append('')
-            rgblight_config.append('#ifndef %s' % (config_key,))
-            rgblight_config.append('#   define %s' % (config_key,))
-            rgblight_config.append('#endif // %s' % (config_key,))
-
-    for json_key, config_key in rgblight_animations.items():
-        if 'animations' in config and config['animations'].get(json_key):
-            rgblight_config.append('')
-            rgblight_config.append('#ifndef %s' % (config_key,))
-            rgblight_config.append('#   define %s' % (config_key,))
-            rgblight_config.append('#endif // %s' % (config_key,))
-
-    return '\n'.join(rgblight_config)
-
-
-def usb_properties(usb_props):
-    """Return the config.h lines that setup USB params.
-    """
-    usb_lines = []
-
-    for info_name, config_name in usb_prop_map.items():
-        if info_name in usb_props:
-            usb_lines.append('')
-            usb_lines.append('#ifndef ' + config_name)
-            usb_lines.append('#    define %s %s' % (config_name, usb_props[info_name]))
-            usb_lines.append('#endif // ' + config_name)
-
-    return '\n'.join(usb_lines)
-
-
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
 @cli.argument('-kb', '--keyboard', help='Keyboard to generate config.h for.')
@@ -228,39 +90,52 @@ def generate_config_h(cli):
         cli.log.error('Invalid keyboard: "%s"', cli.config.generate_config_h.keyboard)
         return False
 
-    # Build the info.json file
-    kb_info_json = info_json(cli.config.generate_config_h.keyboard)
-
     # Build the info_config.h file.
-    config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once']
+    kb_info_json = dotty(info_json(cli.config.generate_config_h.keyboard))
+    info_config_map = _json_load(Path('data/mappings/info_config.json'))
 
-    if 'debounce' in kb_info_json:
-        config_h_lines.append(debounce(kb_info_json['debounce']))
-
-    if 'diode_direction' in kb_info_json:
-        config_h_lines.append(diode_direction(kb_info_json['diode_direction']))
-
-    if 'indicators' in kb_info_json:
-        config_h_lines.append(indicators(kb_info_json['indicators']))
-
-    if 'keyboard_name' in kb_info_json:
-        config_h_lines.append(keyboard_name(kb_info_json['keyboard_name']))
-
-    if 'layout_aliases' in kb_info_json:
-        config_h_lines.append(layout_aliases(kb_info_json['layout_aliases']))
-
-    if 'manufacturer' in kb_info_json:
-        config_h_lines.append(manufacturer(kb_info_json['manufacturer']))
+    config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once']
 
-    if 'rgblight' in kb_info_json:
-        config_h_lines.append(rgblight(kb_info_json['rgblight']))
+    # Iterate through the info_config map to generate basic things
+    for config_key, info_dict in info_config_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
+        to_config = info_dict.get('to_config', True)
+
+        if not to_config:
+            continue
+
+        try:
+            config_value = kb_info_json[info_key]
+        except KeyError:
+            continue
+
+        if key_type.startswith('array'):
+            config_h_lines.append('')
+            config_h_lines.append(f'#ifndef {config_key}')
+            config_h_lines.append(f'#   define {config_key} {{ {", ".join(map(str, config_value))} }}')
+            config_h_lines.append(f'#endif // {config_key}')
+        elif key_type == 'bool':
+            if config_value:
+                config_h_lines.append('')
+                config_h_lines.append(f'#ifndef {config_key}')
+                config_h_lines.append(f'#   define {config_key}')
+                config_h_lines.append(f'#endif // {config_key}')
+        elif key_type == 'mapping':
+            for key, value in config_value.items():
+                config_h_lines.append('')
+                config_h_lines.append(f'#ifndef {key}')
+                config_h_lines.append(f'#   define {key} {value}')
+                config_h_lines.append(f'#endif // {key}')
+        else:
+            config_h_lines.append('')
+            config_h_lines.append(f'#ifndef {config_key}')
+            config_h_lines.append(f'#   define {config_key} {config_value}')
+            config_h_lines.append(f'#endif // {config_key}')
 
     if 'matrix_pins' in kb_info_json:
         config_h_lines.append(matrix_pins(kb_info_json['matrix_pins']))
 
-    if 'usb' in kb_info_json:
-        config_h_lines.append(usb_properties(kb_info_json['usb']))
-
     # Show the results
     config_h = '\n'.join(config_h_lines)
 

lib/python/qmk/cli/generate/rules_mk.py

@@ -1,16 +1,37 @@
 """Used by the make system to generate a rules.mk
 """
+from pathlib import Path
+
+from dotty_dict import dotty
 from milc import cli
 
 from qmk.decorators import automagic_keyboard, automagic_keymap
-from qmk.info import info_json
+from qmk.info import _json_load, info_json
 from qmk.path import is_keyboard, normpath
 
-info_to_rules = {
-    'board': 'BOARD',
-    'bootloader': 'BOOTLOADER',
-    'processor': 'MCU',
-}
+
+def process_mapping_rule(kb_info_json, rules_key, info_dict):
+    """Return the rules.mk line(s) for a mapping rule.
+    """
+    if not info_dict.get('to_c', True):
+        return None
+
+    info_key = info_dict['info_key']
+    key_type = info_dict.get('value_type', 'str')
+
+    try:
+        rules_value = kb_info_json[info_key]
+    except KeyError:
+        return None
+
+    if key_type == 'array':
+        return f'{rules_key} ?= {" ".join(rules_value)}'
+    elif key_type == 'bool':
+        return f'{rules_key} ?= {"on" if rules_value else "off"}'
+    elif key_type == 'mapping':
+        return '\n'.join([f'{key} ?= {value}' for key, value in rules_value.items()])
+
+    return f'{rules_key} ?= {rules_value}'
 
 
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
@@ -22,7 +43,6 @@ info_to_rules = {
 def generate_rules_mk(cli):
     """Generates a rules.mk file from info.json.
     """
-    # Determine our keyboard(s)
     if not cli.config.generate_rules_mk.keyboard:
         cli.log.error('Missing paramater: --keyboard')
         cli.subcommands['info'].print_help()
@@ -32,16 +52,18 @@ def generate_rules_mk(cli):
         cli.log.error('Invalid keyboard: "%s"', cli.config.generate_rules_mk.keyboard)
         return False
 
-    # Build the info.json file
-    kb_info_json = info_json(cli.config.generate_rules_mk.keyboard)
+    kb_info_json = dotty(info_json(cli.config.generate_rules_mk.keyboard))
+    info_rules_map = _json_load(Path('data/mappings/info_rules.json'))
     rules_mk_lines = ['# This file was generated by `qmk generate-rules-mk`. Do not edit or copy.', '']
 
-    # Bring in settings
-    for info_key, rule_key in info_to_rules.items():
-        if info_key in kb_info_json:
-            rules_mk_lines.append(f'{rule_key} ?= {kb_info_json[info_key]}')
+    # Iterate through the info_rules map to generate basic rules
+    for rules_key, info_dict in info_rules_map.items():
+        new_entry = process_mapping_rule(kb_info_json, rules_key, info_dict)
+
+        if new_entry:
+            rules_mk_lines.append(new_entry)
 
-    # Find features that should be enabled
+    # Iterate through features to enable/disable them
     if 'features' in kb_info_json:
         for feature, enabled in kb_info_json['features'].items():
             if feature == 'bootmagic_lite' and enabled:
@@ -51,15 +73,6 @@ def generate_rules_mk(cli):
                 enabled = 'yes' if enabled else 'no'
                 rules_mk_lines.append(f'{feature}_ENABLE ?= {enabled}')
 
-    # Set the LED driver
-    if 'led_matrix' in kb_info_json and 'driver' in kb_info_json['led_matrix']:
-        driver = kb_info_json['led_matrix']['driver']
-        rules_mk_lines.append(f'LED_MATRIX_DRIVER ?= {driver}')
-
-    # Add community layouts
-    if 'community_layouts' in kb_info_json:
-        rules_mk_lines.append(f'LAYOUTS ?= {" ".join(kb_info_json["community_layouts"])}')
-
     # Show the results
     rules_mk = '\n'.join(rules_mk_lines) + '\n'
 
@@ -72,7 +85,7 @@ def generate_rules_mk(cli):
         if cli.args.quiet:
             print(cli.args.output)
         else:
-            cli.log.info('Wrote info_config.h to %s.', cli.args.output)
+            cli.log.info('Wrote rules.mk to %s.', cli.args.output)
 
     else:
         print(rules_mk)

lib/python/qmk/constants.py

@@ -27,7 +27,7 @@ ROW_LETTERS = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnop'
 LED_INDICATORS = {
     'caps_lock': 'LED_CAPS_LOCK_PIN',
     'num_lock': 'LED_NUM_LOCK_PIN',
-    'scrol_lock': 'LED_SCROLL_LOCK_PIN',
+    'scroll_lock': 'LED_SCROLL_LOCK_PIN',
 }
 
 # Constants that should match their counterparts in make

lib/python/qmk/info.py

@@ -5,57 +5,18 @@ from collections.abc import Mapping
 from glob import glob
 from pathlib import Path
 
+import hjson
 import jsonschema
+from dotty_dict import dotty
 from milc import cli
 
-from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS, LED_INDICATORS
+from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
 from qmk.c_parse import find_layouts
 from qmk.keyboard import config_h, rules_mk
 from qmk.keymap import list_keymaps
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
 
-led_matrix_properties = {
-    'driver_count': 'LED_DRIVER_COUNT',
-    'driver_addr1': 'LED_DRIVER_ADDR_1',
-    'driver_addr2': 'LED_DRIVER_ADDR_2',
-    'driver_addr3': 'LED_DRIVER_ADDR_3',
-    'driver_addr4': 'LED_DRIVER_ADDR_4',
-    'led_count': 'LED_DRIVER_LED_COUNT',
-    'timeout': 'ISSI_TIMEOUT',
-    'persistence': 'ISSI_PERSISTENCE'
-}
-
-rgblight_properties = {
-    'led_count': ('RGBLED_NUM', int),
-    'pin': ('RGB_DI_PIN', str),
-    'max_brightness': ('RGBLIGHT_LIMIT_VAL', int),
-    'hue_steps': ('RGBLIGHT_HUE_STEP', int),
-    'saturation_steps': ('RGBLIGHT_SAT_STEP', int),
-    'brightness_steps': ('RGBLIGHT_VAL_STEP', int)
-}
-
-rgblight_toggles = {
-    'sleep': 'RGBLIGHT_SLEEP',
-    'split': 'RGBLIGHT_SPLIT',
-}
-
-rgblight_animations = {
-    'all': 'RGBLIGHT_ANIMATIONS',
-    'alternating': 'RGBLIGHT_EFFECT_ALTERNATING',
-    'breathing': 'RGBLIGHT_EFFECT_BREATHING',
-    'christmas': 'RGBLIGHT_EFFECT_CHRISTMAS',
-    'knight': 'RGBLIGHT_EFFECT_KNIGHT',
-    'rainbow_mood': 'RGBLIGHT_EFFECT_RAINBOW_MOOD',
-    'rainbow_swirl': 'RGBLIGHT_EFFECT_RAINBOW_SWIRL',
-    'rgb_test': 'RGBLIGHT_EFFECT_RGB_TEST',
-    'snake': 'RGBLIGHT_EFFECT_SNAKE',
-    'static_gradient': 'RGBLIGHT_EFFECT_STATIC_GRADIENT',
-    'twinkle': 'RGBLIGHT_EFFECT_TWINKLE'
-}
-
-usb_properties = {'vid': 'VENDOR_ID', 'pid': 'PRODUCT_ID', 'device_ver': 'DEVICE_VER'}
-
 true_values = ['1', 'on', 'yes']
 false_values = ['0', 'off', 'no']
 
@@ -101,7 +62,6 @@ def info_json(keyboard):
     except jsonschema.ValidationError as e:
         json_path = '.'.join([str(p) for p in e.absolute_path])
         cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message)
-        print(dir(e))
         exit()
 
     # Make sure we have at least one layout
@@ -132,7 +92,7 @@ def _json_load(json_file):
     Note: file must be a Path object.
     """
     try:
-        return json.load(json_file.open())
+        return hjson.load(json_file.open())
 
     except json.decoder.JSONDecodeError as e:
         cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e)
@@ -172,65 +132,6 @@ def keyboard_api_validate(data):
     return validator(data)
 
 
-def _extract_debounce(info_data, config_c):
-    """Handle debounce.
-    """
-    if 'debounce' in info_data and 'DEBOUNCE' in config_c:
-        _log_warning(info_data, 'Debounce is specified in both info.json and config.h, the config.h value wins.')
-
-    if 'DEBOUNCE' in config_c:
-        info_data['debounce'] = int(config_c['DEBOUNCE'])
-
-    return info_data
-
-
-def _extract_diode_direction(info_data, config_c):
-    """Handle the diode direction.
-    """
-    if 'diode_direction' in info_data and 'DIODE_DIRECTION' in config_c:
-        _log_warning(info_data, 'Diode direction is specified in both info.json and config.h, the config.h value wins.')
-
-    if 'DIODE_DIRECTION' in config_c:
-        info_data['diode_direction'] = config_c.get('DIODE_DIRECTION')
-
-    return info_data
-
-
-def _extract_indicators(info_data, config_c):
-    """Find the LED indicator information.
-    """
-    for json_key, config_key in LED_INDICATORS.items():
-        if json_key in info_data.get('indicators', []) and config_key in config_c:
-            _log_warning(info_data, f'Indicator {json_key} is specified in both info.json and config.h, the config.h value wins.')
-
-        if 'indicators' not in info_data:
-            info_data['indicators'] = {}
-
-        if config_key in config_c:
-            if 'indicators' not in info_data:
-                info_data['indicators'] = {}
-
-            info_data['indicators'][json_key] = config_c.get(config_key)
-
-    return info_data
-
-
-def _extract_community_layouts(info_data, rules):
-    """Find the community layouts in rules.mk.
-    """
-    community_layouts = rules['LAYOUTS'].split() if 'LAYOUTS' in rules else []
-
-    if 'community_layouts' in info_data:
-        for layout in community_layouts:
-            if layout not in info_data['community_layouts']:
-                community_layouts.append(layout)
-
-    else:
-        info_data['community_layouts'] = community_layouts
-
-    return info_data
-
-
 def _extract_features(info_data, rules):
     """Find all the features enabled in rules.mk.
     """
@@ -267,78 +168,6 @@ def _extract_features(info_data, rules):
     return info_data
 
 
-def _extract_led_drivers(info_data, rules):
-    """Find all the LED drivers set in rules.mk.
-    """
-    if 'LED_MATRIX_DRIVER' in rules:
-        if 'led_matrix' not in info_data:
-            info_data['led_matrix'] = {}
-
-        if info_data['led_matrix'].get('driver'):
-            _log_warning(info_data, 'LED Matrix driver is specified in both info.json and rules.mk, the rules.mk value wins.')
-
-        info_data['led_matrix']['driver'] = rules['LED_MATRIX_DRIVER']
-
-    return info_data
-
-
-def _extract_led_matrix(info_data, config_c):
-    """Handle the led_matrix configuration.
-    """
-    led_matrix = info_data.get('led_matrix', {})
-
-    for json_key, config_key in led_matrix_properties.items():
-        if config_key in config_c:
-            if json_key in led_matrix:
-                _log_warning(info_data, 'LED Matrix: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,))
-
-            led_matrix[json_key] = config_c[config_key]
-
-
-def _extract_rgblight(info_data, config_c):
-    """Handle the rgblight configuration.
-    """
-    rgblight = info_data.get('rgblight', {})
-    animations = rgblight.get('animations', {})
-
-    if 'RGBLED_SPLIT' in config_c:
-        raw_split = config_c.get('RGBLED_SPLIT', '').replace('{', '').replace('}', '').strip()
-        rgblight['split_count'] = [int(i) for i in raw_split.split(',')]
-
-    for json_key, config_key_type in rgblight_properties.items():
-        config_key, config_type = config_key_type
-
-        if config_key in config_c:
-            if json_key in rgblight:
-                _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,))
-
-            try:
-                rgblight[json_key] = config_type(config_c[config_key])
-            except ValueError as e:
-                cli.log.error('%s: config.h: Could not convert "%s" to %s: %s', info_data['keyboard_folder'], config_c[config_key], config_type.__name__, e)
-
-    for json_key, config_key in rgblight_toggles.items():
-        if config_key in config_c and json_key in rgblight:
-            _log_warning(info_data, 'RGB Light: %s is specified in both info.json and config.h, the config.h value wins.', json_key)
-
-        rgblight[json_key] = config_key in config_c
-
-    for json_key, config_key in rgblight_animations.items():
-        if config_key in config_c:
-            if json_key in animations:
-                _log_warning(info_data, 'RGB Light: animations: %s is specified in both info.json and config.h, the config.h value wins.' % (json_key,))
-
-            animations[json_key] = config_c[config_key]
-
-    if animations:
-        rgblight['animations'] = animations
-
-    if rgblight:
-        info_data['rgblight'] = rgblight
-
-    return info_data
-
-
 def _pin_name(pin):
     """Returns the proper representation for a pin.
     """
@@ -426,34 +255,59 @@ def _extract_matrix_info(info_data, config_c):
     return info_data
 
 
-def _extract_usb_info(info_data, config_c):
-    """Populate the USB information.
+def _extract_config_h(info_data):
+    """Pull some keyboard information from existing config.h files
     """
-    if 'usb' not in info_data:
-        info_data['usb'] = {}
+    config_c = config_h(info_data['keyboard_folder'])
 
-    for info_name, config_name in usb_properties.items():
-        if config_name in config_c:
-            if info_name in info_data['usb']:
-                _log_warning(info_data, '%s in config.h is overwriting usb.%s in info.json' % (config_name, info_name))
+    # Pull in data from the json map
+    dotty_info = dotty(info_data)
+    info_config_map = _json_load(Path('data/mappings/info_config.json'))
 
-            info_data['usb'][info_name] = '0x' + config_c[config_name][2:].upper()
+    for config_key, info_dict in info_config_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
 
-    return info_data
+        try:
+            if config_key in config_c and info_dict.get('to_json', True):
+                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
+                    _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
 
+                if key_type.startswith('array'):
+                    if '.' in key_type:
+                        key_type, array_type = key_type.split('.', 1)
+                    else:
+                        array_type = None
 
-def _extract_config_h(info_data):
-    """Pull some keyboard information from existing config.h files
-    """
-    config_c = config_h(info_data['keyboard_folder'])
+                    config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
+
+                    if array_type == 'int':
+                        dotty_info[info_key] = list(map(int, config_value.split(',')))
+                    else:
+                        dotty_info[info_key] = config_value.split(',')
+
+                elif key_type == 'bool':
+                    dotty_info[info_key] = config_c[config_key] in true_values
+
+                elif key_type == 'hex':
+                    dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
 
-    _extract_debounce(info_data, config_c)
-    _extract_diode_direction(info_data, config_c)
-    _extract_indicators(info_data, config_c)
+                elif key_type == 'list':
+                    dotty_info[info_key] = config_c[config_key].split()
+
+                elif key_type == 'int':
+                    dotty_info[info_key] = int(config_c[config_key])
+
+                else:
+                    dotty_info[info_key] = config_c[config_key]
+
+        except Exception as e:
+            _log_warning(info_data, f'{config_key}->{info_key}: {e}')
+
+    info_data.update(dotty_info)
+
+    # Pull data that easily can't be mapped in json
     _extract_matrix_info(info_data, config_c)
-    _extract_usb_info(info_data, config_c)
-    _extract_led_matrix(info_data, config_c)
-    _extract_rgblight(info_data, config_c)
 
     return info_data
 
@@ -462,21 +316,66 @@ def _extract_rules_mk(info_data):
     """Pull some keyboard information from existing rules.mk files
     """
     rules = rules_mk(info_data['keyboard_folder'])
-    mcu = rules.get('MCU', info_data.get('processor'))
+    info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
 
-    if mcu in CHIBIOS_PROCESSORS:
+    if info_data['processor'] in CHIBIOS_PROCESSORS:
         arm_processor_rules(info_data, rules)
 
-    elif mcu in LUFA_PROCESSORS + VUSB_PROCESSORS:
+    elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
         avr_processor_rules(info_data, rules)
 
     else:
-        cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], mcu))
+        cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
         unknown_processor_rules(info_data, rules)
 
-    _extract_community_layouts(info_data, rules)
+    # Pull in data from the json map
+    dotty_info = dotty(info_data)
+    info_rules_map = _json_load(Path('data/mappings/info_rules.json'))
+
+    for rules_key, info_dict in info_rules_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
+
+        try:
+            if rules_key in rules and info_dict.get('to_json', True):
+                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
+                    _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
+
+                if key_type.startswith('array'):
+                    if '.' in key_type:
+                        key_type, array_type = key_type.split('.', 1)
+                    else:
+                        array_type = None
+
+                    rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
+
+                    if array_type == 'int':
+                        dotty_info[info_key] = list(map(int, rules_value.split(',')))
+                    else:
+                        dotty_info[info_key] = rules_value.split(',')
+
+                elif key_type == 'list':
+                    dotty_info[info_key] = rules[rules_key].split()
+
+                elif key_type == 'bool':
+                    dotty_info[info_key] = rules[rules_key] in true_values
+
+                elif key_type == 'hex':
+                    dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
+
+                elif key_type == 'int':
+                    dotty_info[info_key] = int(rules[rules_key])
+
+                else:
+                    dotty_info[info_key] = rules[rules_key]
+
+        except Exception as e:
+            _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
+
+    info_data.update(dotty_info)
+
+    # Merge in config values that can't be easily mapped
     _extract_features(info_data, rules)
-    _extract_led_drivers(info_data, rules)
 
     return info_data
 
@@ -565,23 +464,7 @@ def arm_processor_rules(info_data, rules):
     info_data['processor_type'] = 'arm'
     info_data['protocol'] = 'ChibiOS'
 
-    if 'MCU' in rules:
-        if 'processor' in info_data:
-            _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.')
-
-        info_data['processor'] = rules['MCU']
-
-    elif 'processor' not in info_data:
-        info_data['processor'] = 'unknown'
-
-    if 'BOOTLOADER' in rules:
-        # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first
-        # if 'bootloader' in info_data:
-        #     _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.')
-
-        info_data['bootloader'] = rules['BOOTLOADER']
-
-    else:
+    if 'bootloader' not in info_data:
         if 'STM32' in info_data['processor']:
             info_data['bootloader'] = 'stm32-dfu'
         else:
@@ -594,12 +477,6 @@ def arm_processor_rules(info_data, rules):
     elif 'ARM_ATSAM' in rules:
         info_data['platform'] = 'ARM_ATSAM'
 
-    if 'BOARD' in rules:
-        if 'board' in info_data:
-            _log_warning(info_data, 'Board is specified in both info.json and rules.mk, the rules.mk value wins.')
-
-        info_data['board'] = rules['BOARD']
-
     return info_data
 
 
@@ -607,26 +484,10 @@ def avr_processor_rules(info_data, rules):
     """Setup the default info for an AVR board.
     """
     info_data['processor_type'] = 'avr'
-    info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'atmel-dfu'
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
     info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
 
-    if 'MCU' in rules:
-        if 'processor' in info_data:
-            _log_warning(info_data, 'Processor/MCU is specified in both info.json and rules.mk, the rules.mk value wins.')
-
-        info_data['processor'] = rules['MCU']
-
-    elif 'processor' not in info_data:
-        info_data['processor'] = 'unknown'
-
-    if 'BOOTLOADER' in rules:
-        # FIXME(skullydazed/anyone): need to remove the massive amounts of duplication first
-        # if 'bootloader' in info_data:
-        #     _log_warning(info_data, 'Bootloader is specified in both info.json and rules.mk, the rules.mk value wins.')
-
-        info_data['bootloader'] = rules['BOOTLOADER']
-    else:
+    if 'bootloader' not in info_data:
         info_data['bootloader'] = 'atmel-dfu'
 
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:

lib/python/qmk/tests/test_cli_commands.py

@@ -230,3 +230,32 @@ def test_generate_rgb_breathe_table():
     check_returncode(result)
     assert 'Breathing center: 1.2' in result.stdout
     assert 'Breathing max:    127' in result.stdout
+
+
+def test_generate_config_h():
+    result = check_subcommand('generate-config-h', '-kb', 'handwired/pytest/basic')
+    check_returncode(result)
+    assert '#   define DEVICE_VER 0x0001' in result.stdout
+    assert '#   define DESCRIPTION handwired/pytest/basic' in result.stdout
+    assert '#   define DIODE_DIRECTION COL2ROW' in result.stdout
+    assert '#   define MANUFACTURER none' in result.stdout
+    assert '#   define PRODUCT handwired/pytest/basic' in result.stdout
+    assert '#   define PRODUCT_ID 0x6465' in result.stdout
+    assert '#   define VENDOR_ID 0xFEED' in result.stdout
+    assert '#   define MATRIX_COLS 1' in result.stdout
+    assert '#   define MATRIX_COL_PINS { F4 }' in result.stdout
+    assert '#   define MATRIX_ROWS 1' in result.stdout
+    assert '#   define MATRIX_ROW_PINS { F5 }' in result.stdout
+
+
+def test_generate_rules_mk():
+    result = check_subcommand('generate-rules-mk', '-kb', 'handwired/pytest/basic')
+    check_returncode(result)
+    assert 'BOOTLOADER ?= atmel-dfu' in result.stdout
+    assert 'MCU ?= atmega32u4' in result.stdout
+
+
+def test_generate_layouts():
+    result = check_subcommand('generate-layouts', '-kb', 'handwired/pytest/basic')
+    check_returncode(result)
+    assert '#define LAYOUT_custom(k0A) {' in result.stdout

requirements.txt

@@ -2,6 +2,7 @@
 appdirs
 argcomplete
 colorama
+dotty-dict
 hjson
 jsonschema
 milc