QMK Userspace (#22222)

Co-authored-by: Duncan Sutherland <dunk2k_2000@hotmail.com>

Makefile

@@ -38,6 +38,11 @@ $(info QMK Firmware $(QMK_VERSION))
 endif
 endif
 
+# Try to determine userspace from qmk config, if set.
+ifeq ($(QMK_USERSPACE),)
+    QMK_USERSPACE = $(shell qmk config -ro user.overlay_dir | cut -d= -f2 | sed -e 's@^None$$@@g')
+endif
+
 # Determine which qmk cli to use
 QMK_BIN := qmk
 
@@ -191,9 +196,20 @@ define PARSE_KEYBOARD
     KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.)))
     KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.)))
 
+    ifneq ($(QMK_USERSPACE),)
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_1)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_2)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_3)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_4)/keymaps/*/.)))
+        KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/keyboards/$$(KEYBOARD_FOLDER_PATH_5)/keymaps/*/.)))
+    endif
+
     KEYBOARD_LAYOUTS := $(shell $(QMK_BIN) list-layouts --keyboard $1)
     LAYOUT_KEYMAPS :=
     $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(ROOT_DIR)/layouts/*/$$(LAYOUT)/*/.)))))
+    ifneq ($(QMK_USERSPACE),)
+        $$(foreach LAYOUT,$$(KEYBOARD_LAYOUTS),$$(eval LAYOUT_KEYMAPS += $$(notdir $$(patsubst %/.,%,$$(wildcard $(QMK_USERSPACE)/layouts/$$(LAYOUT)/*/.)))))
+    endif
 
     KEYMAPS := $$(sort $$(KEYMAPS) $$(LAYOUT_KEYMAPS))
 
@@ -431,8 +447,18 @@ clean:
 	rm -rf $(BUILD_DIR)
 	echo 'done.'
 
-.PHONY: distclean
-distclean: clean
+.PHONY: distclean distclean_qmk
+distclean: distclean_qmk
+distclean_qmk: clean
 	echo -n 'Deleting *.bin, *.hex, and *.uf2 ... '
 	rm -f *.bin *.hex *.uf2
 	echo 'done.'
+
+ifneq ($(QMK_USERSPACE),)
+.PHONY: distclean_userspace
+distclean: distclean_userspace
+distclean_userspace: clean
+	echo -n 'Deleting userspace *.bin, *.hex, and *.uf2 ... '
+	rm -f $(QMK_USERSPACE)/*.bin $(QMK_USERSPACE)/*.hex $(QMK_USERSPACE)/*.uf2
+	echo 'done.'
+endif

builddefs/build_json.mk

@@ -15,3 +15,22 @@ else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
     KEYMAP_JSON_PATH := $(MAIN_KEYMAP_PATH_1)
 endif
+
+ifneq ($(QMK_USERSPACE),)
+    ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)
+    else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.json)","")
+        KEYMAP_JSON := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.json
+        KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)
+    endif
+endif

builddefs/build_keyboard.mk

@@ -127,34 +127,60 @@ include $(INFO_RULES_MK)
 include $(BUILDDEFS_PATH)/build_json.mk
 
 # Pull in keymap level rules.mk
-# Look through the possible keymap folders until we find a matching keymap.c
-ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_1)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_2)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_3)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_4)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
-else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
-    -include $(MAIN_KEYMAP_PATH_5)/rules.mk
-    KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
-    KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
-else ifneq ($(LAYOUTS),)
-    # If we haven't found a keymap yet fall back to community layouts
-    include $(BUILDDEFS_PATH)/build_layout.mk
-# Not finding keymap.c is fine if we found a keymap.json
-else ifeq ("$(wildcard $(KEYMAP_JSON_PATH))", "")
-    $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
-    # this state should never be reached
+ifeq ("$(wildcard $(KEYMAP_PATH))", "")
+    # Look through the possible keymap folders until we find a matching keymap.c
+    ifneq ($(QMK_USERSPACE),)
+        ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_1)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_2)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_3)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_4)
+        else ifneq ("$(wildcard $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+            -include $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/rules.mk
+            KEYMAP_C := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)/keymap.c
+            KEYMAP_PATH := $(QMK_USERSPACE)/$(MAIN_KEYMAP_PATH_5)
+        endif
+    endif
+    ifeq ($(KEYMAP_PATH),)
+        ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_1)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_1)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_2)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_2)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_3)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_3)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_4)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_4)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
+        else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.c)","")
+            -include $(MAIN_KEYMAP_PATH_5)/rules.mk
+            KEYMAP_C := $(MAIN_KEYMAP_PATH_5)/keymap.c
+            KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
+        else ifneq ($(LAYOUTS),)
+            # If we haven't found a keymap yet fall back to community layouts
+            include $(BUILDDEFS_PATH)/build_layout.mk
+        else ifeq ("$(wildcard $(KEYMAP_JSON_PATH))", "") # Not finding keymap.c is fine if we found a keymap.json
+            $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
+            # this state should never be reached
+        endif
+    endif
 endif
 
 # Have we found a keymap.json?
@@ -364,6 +390,16 @@ ifeq ("$(USER_NAME)","")
 endif
 USER_PATH := users/$(USER_NAME)
 
+# If we have userspace, then add it to the lookup VPATH
+ifneq ($(wildcard $(QMK_USERSPACE)),)
+    VPATH += $(QMK_USERSPACE)
+endif
+
+# If the equivalent users directory exists in userspace, use that in preference to anything currently in the main repo
+ifneq ($(wildcard $(QMK_USERSPACE)/$(USER_PATH)),)
+    USER_PATH := $(QMK_USERSPACE)/$(USER_PATH)
+endif
+
 # Pull in user level rules.mk
 -include $(USER_PATH)/rules.mk
 ifneq ("$(wildcard $(USER_PATH)/config.h)","")
@@ -404,6 +440,10 @@ ifneq ("$(KEYMAP_H)","")
     CONFIG_H += $(KEYMAP_H)
 endif
 
+ifeq ($(KEYMAP_C),)
+    $(call CATASTROPHIC_ERROR,Invalid keymap,Could not find keymap)
+endif
+
 OPT_DEFS += -DKEYMAP_C=\"$(KEYMAP_C)\"
 
 # If a keymap or userspace places their keymap array in another file instead, allow for it to be included

builddefs/build_layout.mk

@@ -1,6 +1,10 @@
 LAYOUTS_PATH := layouts
 LAYOUTS_REPOS := $(patsubst %/,%,$(sort $(dir $(wildcard $(LAYOUTS_PATH)/*/))))
 
+ifneq ($(QMK_USERSPACE),)
+    LAYOUTS_REPOS += $(patsubst %/,%,$(QMK_USERSPACE)/$(LAYOUTS_PATH))
+endif
+
 define SEARCH_LAYOUTS_REPO
     LAYOUT_KEYMAP_PATH := $$(LAYOUTS_REPO)/$$(LAYOUT)/$$(KEYMAP)
     LAYOUT_KEYMAP_JSON := $$(LAYOUT_KEYMAP_PATH)/keymap.json

builddefs/common_rules.mk

@@ -191,7 +191,7 @@ DFU_SUFFIX_ARGS ?=
 elf: $(BUILD_DIR)/$(TARGET).elf
 hex: $(BUILD_DIR)/$(TARGET).hex
 uf2: $(BUILD_DIR)/$(TARGET).uf2
-cpfirmware: $(FIRMWARE_FORMAT)
+cpfirmware_qmk: $(FIRMWARE_FORMAT)
 	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to qmk_firmware folder" | $(AWK_CMD)
 	$(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
 eep: $(BUILD_DIR)/$(TARGET).eep
@@ -200,6 +200,15 @@ sym: $(BUILD_DIR)/$(TARGET).sym
 LIBNAME=lib$(TARGET).a
 lib: $(LIBNAME)
 
+cpfirmware: cpfirmware_qmk
+
+ifneq ($(QMK_USERSPACE),)
+cpfirmware: cpfirmware_userspace
+cpfirmware_userspace: cpfirmware_qmk
+	$(SILENT) || printf "Copying $(TARGET).$(FIRMWARE_FORMAT) to userspace folder" | $(AWK_CMD)
+	$(COPY) $(BUILD_DIR)/$(TARGET).$(FIRMWARE_FORMAT) $(QMK_USERSPACE)/$(TARGET).$(FIRMWARE_FORMAT) && $(PRINT_OK)
+endif
+
 # Display size of file, modifying the output so people don't mistakenly grab the hex output
 BINARY_SIZE = $(SIZE) --target=$(FORMAT) $(BUILD_DIR)/$(TARGET).hex | $(SED) -e 's/\.build\/.*$$/$(TARGET).$(FIRMWARE_FORMAT)/g'
 

data/schemas/definitions.jsonschema

@@ -177,5 +177,23 @@
         "type": "integer",
         "minimum": 0,
         "maximum": 1
+    },
+    "keyboard_keymap_tuple": {
+        "type": "array",
+        "prefixItems": [
+            { "$ref": "#/keyboard" },
+            { "$ref": "#/filename" }
+        ],
+        "unevaluatedItems": false
+    },
+    "json_file_path": {
+        "type": "string",
+        "pattern": "^[0-9a-z_/\\-]+\\.json$"
+    },
+    "build_target": {
+        "oneOf": [
+            { "$ref": "#/keyboard_keymap_tuple" },
+            { "$ref": "#/json_file_path" }
+        ]
     }
 }

data/schemas/user_repo_v0.jsonschema

@@ -0,0 +1,14 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema#",
+    "$id": "qmk.user_repo.v0",
+    "title": "User Repository Information",
+    "type": "object",
+    "required": [
+        "userspace_version"
+    ],
+    "properties": {
+        "userspace_version": {
+            "type": "string",
+        },
+    }
+}

data/schemas/user_repo_v1.jsonschema

@@ -0,0 +1,22 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema#",
+    "$id": "qmk.user_repo.v1",
+    "title": "User Repository Information",
+    "type": "object",
+    "required": [
+        "userspace_version",
+        "build_targets"
+    ],
+    "properties": {
+        "userspace_version": {
+            "type": "string",
+            "enum": ["1.0"]
+        },
+        "build_targets": {
+            "type": "array",
+            "items": {
+                "$ref": "qmk.definitions.v1#/build_target"
+            }
+        }
+    }
+}

docs/_summary.md

@@ -4,7 +4,7 @@
   * [Building Your First Firmware](newbs_building_firmware.md)
   * [Flashing Firmware](newbs_flashing.md)
   * [Getting Help/Support](support.md)
-  * [Building With GitHub Userspace](newbs_building_firmware_workflow.md)
+  * [External Userspace](newbs_external_userspace.md)
   * [Other Resources](newbs_learn_more_resources.md)
   * [Syllabus](syllabus.md)
 

docs/cli_commands.md

@@ -482,6 +482,131 @@ $ qmk import-kbfirmware ~/Downloads/gh62.json
 
 ---
 
+# External Userspace Commands
+
+## `qmk userspace-add`
+
+This command adds a keyboard/keymap to the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-add [-h] [-km KEYMAP] [-kb KEYBOARD] [builds ...]
+
+positional arguments:
+  builds                List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.
+
+options:
+  -h, --help            show this help message and exit
+  -km KEYMAP, --keymap KEYMAP
+                        The keymap to build a firmware for. Ignored when a configurator export is supplied.
+  -kb KEYBOARD, --keyboard KEYBOARD
+                        The keyboard to build a firmware for. Ignored when a configurator export is supplied.
+```
+
+**Example**:
+
+```
+$ qmk userspace-add -kb planck/rev6 -km default
+Ψ Added planck/rev6:default to userspace build targets
+Ψ Saved userspace file to /home/you/qmk_userspace/qmk.json
+```
+
+## `qmk userspace-remove`
+
+This command removes a keyboard/keymap from the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-remove [-h] [-km KEYMAP] [-kb KEYBOARD] [builds ...]
+
+positional arguments:
+  builds                List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.
+
+options:
+  -h, --help            show this help message and exit
+  -km KEYMAP, --keymap KEYMAP
+                        The keymap to build a firmware for. Ignored when a configurator export is supplied.
+  -kb KEYBOARD, --keyboard KEYBOARD
+                        The keyboard to build a firmware for. Ignored when a configurator export is supplied.
+```
+
+**Example**:
+
+```
+$ qmk userspace-remove -kb planck/rev6 -km default
+Ψ Removed planck/rev6:default from userspace build targets
+Ψ Saved userspace file to /home/you/qmk_userspace/qmk.json
+```
+
+## `qmk userspace-list`
+
+This command lists the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-list [-h] [-e]
+
+options:
+  -h, --help    show this help message and exit
+  -e, --expand  Expands any use of `all` for either keyboard or keymap.
+```
+
+**Example**:
+
+```
+$ qmk userspace-list
+Ψ Current userspace build targets:
+Ψ Keyboard: planck/rev6, keymap: you
+Ψ Keyboard: clueboard/66/rev3, keymap: you
+```
+
+## `qmk userspace-compile`
+
+This command compiles all the External Userspace build targets.
+
+**Usage**:
+
+```
+qmk userspace-compile [-h] [-e ENV] [-n] [-c] [-j PARALLEL] [-t]
+
+options:
+  -h, --help            show this help message and exit
+  -e ENV, --env ENV     Set a variable to be passed to make. May be passed multiple times.
+  -n, --dry-run         Don't actually build, just show the commands to be run.
+  -c, --clean           Remove object files before compiling.
+  -j PARALLEL, --parallel PARALLEL
+                        Set the number of parallel make jobs; 0 means unlimited.
+  -t, --no-temp         Remove temporary files during build.
+```
+
+**Example**:
+
+```
+$ qmk userspace-compile
+Ψ Preparing target list...
+Build planck/rev6:you                                                  [OK]
+Build clueboard/66/rev3:you                                            [OK]
+```
+
+## `qmk userspace-doctor`
+
+This command examines your environment and alerts you to potential problems related to External Userspace.
+
+**Example**:
+
+```
+% qmk userspace-doctor
+Ψ QMK home: /home/you/qmk_userspace/qmk_firmware
+Ψ Testing userspace candidate: /home/you/qmk_userspace -- Valid `qmk.json`
+Ψ QMK userspace: /home/you/qmk_userspace
+Ψ Userspace enabled: True
+```
+
+---
+
 # Developer Commands
 
 ## `qmk format-text`

docs/newbs_external_userspace.md

@@ -0,0 +1,96 @@
+# External QMK Userspace
+
+QMK Firmware now officially supports storing user keymaps outside of the normal QMK Firmware repository, allowing users to maintain their own keymaps without having to fork, modify, and maintain a copy of QMK Firmware themselves.
+
+External Userspace mirrors the structure of the main QMK Firmware repository, but only contains the keymaps that you wish to build. You can still use `keyboards/<my keyboard>/keymaps/<my keymap>` to store your keymaps, or you can use the `layouts/<my layout>/<my keymap>` system as before -- they're just stored external to QMK Firmware.
+
+The build system will still honor the use of `users/<my keymap>` if you rely on the traditional QMK Firmware [userspace feature](feature_userspace.md) -- it's now supported externally too, using the same location inside the External Userspace directory.
+
+Additionally, there is first-class support for using GitHub Actions to build your keymaps, allowing you to automatically compile your keymaps whenever you push changes to your External Userspace repository.
+
+!> External Userspace is new functionality and may have issues. Tighter integration with the `qmk` command will occur over time.
+
+?> Historical keymap.json and GitHub-based firmware build instructions can be found [here](newbs_building_firmware_workflow.md). This document supersedes those instructions, but they should still function correctly.
+
+## Setting up QMK Locally
+
+If you wish to build on your local machine, you will need to set up QMK locally. This is a one-time process, and is documented in the [newbs setup guide](https://docs.qmk.fm/#/newbs).
+
+!> If you wish to use any QMK CLI commands related to manipulating External Userspace definitions, you will currently need a copy of QMK Firmware as well.
+
+!> Building locally has a much shorter turnaround time than waiting for GitHub Actions to complete.
+
+## External Userspace Repository Setup (forked on GitHub)
+
+A basic skeleton External Userspace repository can be found [here](https://github.com/qmk/qmk_userspace). If you wish to keep your keymaps on GitHub (strongly recommended!), you can fork the repository and use it as a base:
+
+![Userspace Fork](https://i.imgur.com/hcegguh.png)
+
+Going ahead with your fork will copy it to your account, at which point you can clone it to your local machine and begin adding your keymaps:
+
+![Userspace Clone](https://i.imgur.com/CWYmsk8.png)
+
+```sh
+cd $HOME
+git clone https://github.com/{myusername}/qmk_userspace.git
+qmk config user.overlay_dir="$(realpath qmk_userspace)"
+```
+
+## External Userspace Setup (locally stored only)
+
+If you don't want to use GitHub and prefer to keep everything local, you can clone a copy of the default External Userspace locally instead:
+
+```sh
+cd $HOME
+git clone https://github.com/qmk/qmk_userspace.git
+qmk config user.overlay_dir="$(realpath qmk_userspace)"
+```
+
+## Adding a Keymap
+
+_These instructions assume you have already set up QMK locally, and have a copy of the QMK Firmware repository on your machine._
+
+Keymaps within External Userspace are defined in the same way as they are in the main QMK repository. You can either use the `qmk new-keymap` command to create a new keymap, or manually create a new directory in the `keyboards` directory.
+
+Alternatively, you can use the `layouts` directory to store your keymaps, using the same layout system as the main QMK repository -- if you choose to do so you'll want to use the path `layouts/<layout name>/<keymap name>/keymap.*` to store your keymap files, where `layout name` matches an existing layout in QMK, such as `tkl_ansi`.
+
+After creating your new keymap, building the keymap matches normal QMK usage:
+
+```sh
+qmk compile -kb <keyboard> -km <keymap>
+```
+
+!> The `qmk config user.overlay_dir=...` command must have been run when cloning the External Userspace repository for this to work correctly.
+
+## Adding the keymap to External Userspace build targets
+
+Once you have created your keymap, if you want to use GitHub Actions to build your firmware, you will need to add it to the External Userspace build targets. This is done using the `qmk userspace-add` command:
+
+```sh
+# for a keyboard/keymap combo:
+qmk userspace-add -kb <keyboard> -km <keymap>
+# or, for a json-based keymap (if kept "loose"):
+qmk userspace-add <relative/path/to/my/keymap.json>
+```
+
+This updates the `qmk.json` file in the root of your External Userspace directory. If you're using a git repository to store your keymaps, now is a great time to commit and push to your own fork.
+
+## Compiling External Userspace build targets
+
+Once you have added your keymaps to the External Userspace build targets, you can compile all of them at once using the `qmk userspace-compile` command:
+
+```sh
+qmk userspace-compile
+```
+
+All firmware builds you've added to the External Userspace build targets will be built, and the resulting firmware files will be placed in the root of your External Userspace directory.
+
+## Using GitHub Actions
+
+GitHub Actions can be used to automatically build your keymaps whenever you push changes to your External Userspace repository. If you have set up your list of build targets, this is as simple as enabling workflows in the GitHub repository settings:
+
+![Repo Settings](https://i.imgur.com/EVkxOt1.png)
+
+Any push will result in compilation of all configured builds, and once completed a new release containing the newly-minted firmware files will be created on GitHub, which you can subsequently download and flash to your keyboard:
+
+![Releases](https://i.imgur.com/zmwOL5P.png)

lib/python/qmk/build_targets.py

@@ -10,6 +10,8 @@ from qmk.constants import QMK_FIRMWARE, INTERMEDIATE_OUTPUT_PREFIX
 from qmk.commands import find_make, get_make_parallel_args, parse_configurator_json
 from qmk.keyboard import keyboard_folder
 from qmk.info import keymap_json
+from qmk.keymap import locate_keymap
+from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace
 
 
 class BuildTarget:
@@ -158,6 +160,20 @@ class KeyboardKeymapBuildTarget(BuildTarget):
         for key, value in env_vars.items():
             compile_args.append(f'{key}={value}')
 
+        # Need to override the keymap path if the keymap is a userspace directory.
+        # This also ensures keyboard aliases as per `keyboard_aliases.hjson` still work if the userspace has the keymap
+        # in an equivalent historical location.
+        keymap_location = locate_keymap(self.keyboard, self.keymap)
+        if is_under_qmk_userspace(keymap_location) and not is_under_qmk_firmware(keymap_location):
+            keymap_directory = keymap_location.parent
+            compile_args.extend([
+                f'MAIN_KEYMAP_PATH_1={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_2={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_3={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_4={keymap_directory}',
+                f'MAIN_KEYMAP_PATH_5={keymap_directory}',
+            ])
+
         return compile_args
 
 

lib/python/qmk/cli/__init__.py

@@ -81,6 +81,11 @@ subcommands = [
     'qmk.cli.new.keymap',
     'qmk.cli.painter',
     'qmk.cli.pytest',
+    'qmk.cli.userspace.add',
+    'qmk.cli.userspace.compile',
+    'qmk.cli.userspace.doctor',
+    'qmk.cli.userspace.list',
+    'qmk.cli.userspace.remove',
     'qmk.cli.via2json',
 ]
 

lib/python/qmk/cli/compile.py

@@ -37,7 +37,9 @@ def compile(cli):
         from .mass_compile import mass_compile
         cli.args.builds = []
         cli.args.filter = []
-        cli.args.no_temp = False
+        cli.config.mass_compile.keymap = cli.config.compile.keymap
+        cli.config.mass_compile.parallel = cli.config.compile.parallel
+        cli.config.mass_compile.no_temp = False
         return mass_compile(cli)
 
     # Build the environment vars

lib/python/qmk/cli/doctor/main.py

@@ -9,10 +9,11 @@ from milc import cli
 from milc.questions import yesno
 
 from qmk import submodules
-from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM
+from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM, QMK_USERSPACE, HAS_QMK_USERSPACE
 from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
 from qmk.git import git_check_repo, git_get_branch, git_get_tag, git_get_last_log_entry, git_get_common_ancestor, git_is_dirty, git_get_remotes, git_check_deviation
 from qmk.commands import in_virtualenv
+from qmk.userspace import qmk_userspace_paths, qmk_userspace_validate, UserspaceValidationError
 
 
 def os_tests():
@@ -92,6 +93,25 @@ def output_submodule_status():
                 cli.log.error(f'- {sub_name}: <<< missing or unknown >>>')
 
 
+def userspace_tests(qmk_firmware):
+    if qmk_firmware:
+        cli.log.info(f'QMK home: {{fg_cyan}}{qmk_firmware}')
+
+    for path in qmk_userspace_paths():
+        try:
+            qmk_userspace_validate(path)
+            cli.log.info(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_green}}Valid `qmk.json`')
+        except FileNotFoundError:
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Missing `qmk.json`')
+        except UserspaceValidationError as err:
+            cli.log.warn(f'Testing userspace candidate: {{fg_cyan}}{path}{{fg_reset}} -- {{fg_red}}Invalid `qmk.json`')
+            cli.log.warn(f' -- {{fg_cyan}}{path}/qmk.json{{fg_reset}} validation error: {err}')
+
+    if QMK_USERSPACE is not None:
+        cli.log.info(f'QMK userspace: {{fg_cyan}}{QMK_USERSPACE}')
+    cli.log.info(f'Userspace enabled: {{fg_cyan}}{HAS_QMK_USERSPACE}')
+
+
 @cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')
 @cli.argument('-n', '--no', action='store_true', arg_only=True, help='Answer no to all questions.')
 @cli.subcommand('Basic QMK environment checks')
@@ -108,6 +128,9 @@ def doctor(cli):
     cli.log.info('QMK home: {fg_cyan}%s', QMK_FIRMWARE)
 
     status = os_status = os_tests()
+
+    userspace_tests(None)
+
     git_status = git_tests()
 
     if git_status == CheckStatus.ERROR or (os_status == CheckStatus.OK and git_status == CheckStatus.WARNING):

lib/python/qmk/cli/format/json.py

@@ -9,48 +9,74 @@ from milc import cli
 
 from qmk.info import info_json
 from qmk.json_schema import json_load, validate
-from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
+from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder, UserspaceJSONEncoder
 from qmk.path import normpath
 
 
-@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
-@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
-@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
-@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
-@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
-def format_json(cli):
-    """Format a json file.
+def _detect_json_format(file, json_data):
+    """Detect the format of a json file.
     """
-    json_file = json_load(cli.args.json_file)
-
-    if cli.args.format == 'auto':
+    json_encoder = None
+    try:
+        validate(json_data, 'qmk.user_repo.v1')
+        json_encoder = UserspaceJSONEncoder
+    except ValidationError:
+        pass
+
+    if json_encoder is None:
         try:
-            validate(json_file, 'qmk.keyboard.v1')
+            validate(json_data, 'qmk.keyboard.v1')
             json_encoder = InfoJSONEncoder
-
         except ValidationError as e:
-            cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e)
-            cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
+            cli.log.warning('File %s did not validate as a keyboard info.json or userspace qmk.json:\n\t%s', file, e)
+            cli.log.info('Treating %s as a keymap file.', file)
             json_encoder = KeymapJSONEncoder
+
+    return json_encoder
+
+
+def _get_json_encoder(file, json_data):
+    """Get the json encoder for a file.
+    """
+    json_encoder = None
+    if cli.args.format == 'auto':
+        json_encoder = _detect_json_format(file, json_data)
     elif cli.args.format == 'keyboard':
         json_encoder = InfoJSONEncoder
     elif cli.args.format == 'keymap':
         json_encoder = KeymapJSONEncoder
+    elif cli.args.format == 'userspace':
+        json_encoder = UserspaceJSONEncoder
     else:
         # This should be impossible
         cli.log.error('Unknown format: %s', cli.args.format)
+    return json_encoder
+
+
+@cli.argument('json_file', arg_only=True, type=normpath, help='JSON file to format')
+@cli.argument('-f', '--format', choices=['auto', 'keyboard', 'keymap', 'userspace'], default='auto', arg_only=True, help='JSON formatter to use (Default: autodetect)')
+@cli.argument('-i', '--inplace', action='store_true', arg_only=True, help='If set, will operate in-place on the input file')
+@cli.argument('-p', '--print', action='store_true', arg_only=True, help='If set, will print the formatted json to stdout ')
+@cli.subcommand('Generate an info.json file for a keyboard.', hidden=False if cli.config.user.developer else True)
+def format_json(cli):
+    """Format a json file.
+    """
+    json_data = json_load(cli.args.json_file)
+
+    json_encoder = _get_json_encoder(cli.args.json_file, json_data)
+    if json_encoder is None:
         return False
 
-    if json_encoder == KeymapJSONEncoder and 'layout' in json_file:
+    if json_encoder == KeymapJSONEncoder and 'layout' in json_data:
         # Attempt to format the keycodes.
-        layout = json_file['layout']
-        info_data = info_json(json_file['keyboard'])
+        layout = json_data['layout']
+        info_data = info_json(json_data['keyboard'])
 
         if layout in info_data.get('layout_aliases', {}):
-            layout = json_file['layout'] = info_data['layout_aliases'][layout]
+            layout = json_data['layout'] = info_data['layout_aliases'][layout]
 
         if layout in info_data.get('layouts'):
-            for layer_num, layer in enumerate(json_file['layers']):
+            for layer_num, layer in enumerate(json_data['layers']):
                 current_layer = []
                 last_row = 0
 
@@ -61,9 +87,9 @@ def format_json(cli):
 
                     current_layer.append(keymap_key)
 
-                json_file['layers'][layer_num] = current_layer
+                json_data['layers'][layer_num] = current_layer
 
-    output = json.dumps(json_file, cls=json_encoder, sort_keys=True)
+    output = json.dumps(json_data, cls=json_encoder, sort_keys=True)
 
     if cli.args.inplace:
         with open(cli.args.json_file, 'w+', encoding='utf-8') as outfile:

lib/python/qmk/cli/mass_compile.py

@@ -72,7 +72,7 @@ all: {keyboard_safe}_{keymap_name}_binary
                     # yapf: enable
                 f.write('\n')
 
-        cli.run([make_cmd, *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
+        cli.run([find_make(), *get_make_parallel_args(parallel), '-f', makefile.as_posix(), 'all'], capture_output=False, stdin=DEVNULL)
 
         # Check for failures
         failures = [f for f in builddir.glob(f'failed.log.{os.getpid()}.*')]

lib/python/qmk/cli/new/keymap.py

@@ -5,10 +5,12 @@ import shutil
 from milc import cli
 from milc.questions import question
 
+from qmk.constants import HAS_QMK_USERSPACE, QMK_USERSPACE
 from qmk.path import is_keyboard, keymaps, keymap
 from qmk.git import git_get_username
 from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.keyboard import keyboard_completer, keyboard_folder
+from qmk.userspace import UserspaceDefs
 
 
 def prompt_keyboard():
@@ -68,3 +70,9 @@ def new_keymap(cli):
     # end message to user
     cli.log.info(f'{{fg_green}}Created a new keymap called {{fg_cyan}}{user_name}{{fg_green}} in: {{fg_cyan}}{keymap_path_new}.{{fg_reset}}')
     cli.log.info(f"Compile a firmware with your new keymap by typing: {{fg_yellow}}qmk compile -kb {kb_name} -km {user_name}{{fg_reset}}.")
+
+    # Add to userspace compile if we have userspace available
+    if HAS_QMK_USERSPACE:
+        userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+        userspace.add_target(keyboard=kb_name, keymap=user_name, do_print=False)
+        return userspace.save()

lib/python/qmk/cli/userspace/__init__.py

@@ -0,0 +1,5 @@
+from . import doctor
+from . import add
+from . import remove
+from . import list
+from . import compile

lib/python/qmk/cli/userspace/add.py

@@ -0,0 +1,51 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
+from qmk.keymap import keymap_completer, is_keymap_target
+from qmk.userspace import UserspaceDefs
+
+
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
+@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Adds a build target to userspace `qmk.json`.')
+def userspace_add(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    if len(cli.args.builds) > 0:
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+
+        for e in json_like_targets:
+            userspace.add_target(json_path=e)
+
+        for e in make_like_targets:
+            s = e.split(':')
+            userspace.add_target(keyboard=s[0], keymap=s[1])
+
+    else:
+        failed = False
+        try:
+            if not is_keymap_target(cli.args.keyboard, cli.args.keymap):
+                failed = True
+        except KeyError:
+            failed = True
+
+        if failed:
+            from qmk.cli.new.keymap import new_keymap
+            cli.config.new_keymap.keyboard = cli.args.keyboard
+            cli.config.new_keymap.keymap = cli.args.keymap
+            if new_keymap(cli) is not False:
+                userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+        else:
+            userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+
+    return userspace.save()

lib/python/qmk/cli/userspace/compile.py

@@ -0,0 +1,38 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.commands import build_environment
+from qmk.userspace import UserspaceDefs
+from qmk.build_targets import JsonKeymapBuildTarget
+from qmk.search import search_keymap_targets
+from qmk.cli.mass_compile import mass_compile_targets
+
+
+@cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
+@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
+@cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the commands to be run.")
+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.")
+@cli.subcommand('Compiles the build targets specified in userspace `qmk.json`.')
+def userspace_compile(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    build_targets = []
+    keyboard_keymap_targets = []
+    for e in userspace.build_targets:
+        if isinstance(e, Path):
+            build_targets.append(JsonKeymapBuildTarget(e))
+        elif isinstance(e, dict):
+            keyboard_keymap_targets.append((e['keyboard'], e['keymap']))
+
+    if len(keyboard_keymap_targets) > 0:
+        build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
+
+    mass_compile_targets(list(set(build_targets)), cli.args.clean, cli.args.dry_run, cli.config.userspace_compile.no_temp, cli.config.userspace_compile.parallel, **build_environment(cli.args.env))

lib/python/qmk/cli/userspace/doctor.py

@@ -0,0 +1,11 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from milc import cli
+
+from qmk.constants import QMK_FIRMWARE
+from qmk.cli.doctor.main import userspace_tests
+
+
+@cli.subcommand('Checks userspace configuration.')
+def userspace_doctor(cli):
+    userspace_tests(QMK_FIRMWARE)

lib/python/qmk/cli/userspace/list.py

@@ -0,0 +1,51 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from dotty_dict import Dotty
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.userspace import UserspaceDefs
+from qmk.build_targets import BuildTarget
+from qmk.keyboard import is_all_keyboards, keyboard_folder
+from qmk.keymap import is_keymap_target
+from qmk.search import search_keymap_targets
+
+
+@cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.")
+@cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
+def userspace_list(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    if cli.args.expand:
+        build_targets = []
+        for e in userspace.build_targets:
+            if isinstance(e, Path):
+                build_targets.append(e)
+            elif isinstance(e, dict) or isinstance(e, Dotty):
+                build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
+    else:
+        build_targets = userspace.build_targets
+
+    for e in build_targets:
+        if isinstance(e, Path):
+            # JSON keymap from userspace
+            cli.log.info(f'JSON keymap: {{fg_cyan}}{e}{{fg_reset}}')
+            continue
+        elif isinstance(e, dict) or isinstance(e, Dotty):
+            # keyboard/keymap dict from userspace
+            keyboard = e['keyboard']
+            keymap = e['keymap']
+        elif isinstance(e, BuildTarget):
+            # BuildTarget from search_keymap_targets()
+            keyboard = e.keyboard
+            keymap = e.keymap
+
+        if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap):
+            cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
+        else:
+            cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')

lib/python/qmk/cli/userspace/remove.py

@@ -0,0 +1,37 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from pathlib import Path
+from milc import cli
+
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
+from qmk.keymap import keymap_completer
+from qmk.userspace import UserspaceDefs
+
+
+@cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
+@cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
+@cli.subcommand('Removes a build target from userspace `qmk.json`.')
+def userspace_remove(cli):
+    if not HAS_QMK_USERSPACE:
+        cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
+        return False
+
+    userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
+
+    if len(cli.args.builds) > 0:
+        json_like_targets = list([Path(p) for p in filter(lambda e: Path(e).exists() and Path(e).suffix == '.json', cli.args.builds)])
+        make_like_targets = list(filter(lambda e: Path(e) not in json_like_targets, cli.args.builds))
+
+        for e in json_like_targets:
+            userspace.remove_target(json_path=e)
+
+        for e in make_like_targets:
+            s = e.split(':')
+            userspace.remove_target(keyboard=s[0], keymap=s[1])
+
+    else:
+        userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
+
+    return userspace.save()

lib/python/qmk/commands.py

@@ -3,10 +3,12 @@
 import os
 import sys
 import shutil
+from pathlib import Path
 
 from milc import cli
 import jsonschema
 
+from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.json_schema import json_load, validate
 from qmk.keyboard import keyboard_alias_definitions
 
@@ -75,6 +77,10 @@ def build_environment(args):
             envs[key] = value
         else:
             cli.log.warning('Invalid environment variable: %s', env)
+
+    if HAS_QMK_USERSPACE:
+        envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve()
+
     return envs
 
 

lib/python/qmk/constants.py

@@ -4,9 +4,17 @@ from os import environ
 from datetime import date
 from pathlib import Path
 
+from qmk.userspace import detect_qmk_userspace
+
 # The root of the qmk_firmware tree.
 QMK_FIRMWARE = Path.cwd()
 
+# The detected userspace tree
+QMK_USERSPACE = detect_qmk_userspace()
+
+# Whether or not we have a separate userspace directory
+HAS_QMK_USERSPACE = True if QMK_USERSPACE is not None else False
+
 # Upstream repo url
 QMK_FIRMWARE_UPSTREAM = 'qmk/qmk_firmware'
 

lib/python/qmk/json_encoders.py

@@ -217,3 +217,21 @@ class KeymapJSONEncoder(QMKJSONEncoder):
                 return '50' + str(key)
 
         return key
+
+
+class UserspaceJSONEncoder(QMKJSONEncoder):
+    """Custom encoder to make userspace qmk.json's a little nicer to work with.
+    """
+    def sort_dict(self, item):
+        """Sorts the hashes in a nice way.
+        """
+        key = item[0]
+
+        if self.indentation_level == 1:
+            if key == 'userspace_version':
+                return '00userspace_version'
+
+            if key == 'build_targets':
+                return '01build_targets'
+
+        return key

lib/python/qmk/keyboard.py

@@ -78,13 +78,17 @@ def keyboard_alias_definitions():
 def is_all_keyboards(keyboard):
     """Returns True if the keyboard is an AllKeyboards object.
     """
+    if isinstance(keyboard, str):
+        return (keyboard == 'all')
     return isinstance(keyboard, AllKeyboards)
 
 
 def find_keyboard_from_dir():
     """Returns a keyboard name based on the user's current directory.
     """
-    relative_cwd = qmk.path.under_qmk_firmware()
+    relative_cwd = qmk.path.under_qmk_userspace()
+    if not relative_cwd:
+        relative_cwd = qmk.path.under_qmk_firmware()
 
     if relative_cwd and len(relative_cwd.parts) > 1 and relative_cwd.parts[0] == 'keyboards':
         # Attempt to extract the keyboard name from the current directory
@@ -133,6 +137,22 @@ def keyboard_folder(keyboard):
     return keyboard
 
 
+def keyboard_aliases(keyboard):
+    """Returns the list of aliases for the supplied keyboard.
+
+    Includes the keyboard itself.
+    """
+    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
+
+    if keyboard in aliases:
+        keyboard = aliases[keyboard].get('target', keyboard)
+
+    keyboards = set(filter(lambda k: aliases[k].get('target', '') == keyboard, aliases.keys()))
+    keyboards.add(keyboard)
+    keyboards = list(sorted(keyboards))
+    return keyboards
+
+
 def keyboard_folder_or_all(keyboard):
     """Returns the actual keyboard folder.
 

lib/python/qmk/keymap.py

@@ -12,7 +12,8 @@ from pygments.token import Token
 from pygments import lex
 
 import qmk.path
-from qmk.keyboard import find_keyboard_from_dir, keyboard_folder
+from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
+from qmk.keyboard import find_keyboard_from_dir, keyboard_folder, keyboard_aliases
 from qmk.errors import CppError
 from qmk.info import info_json
 
@@ -194,29 +195,38 @@ def _strip_any(keycode):
 def find_keymap_from_dir(*args):
     """Returns `(keymap_name, source)` for the directory provided (or cwd if not specified).
     """
-    relative_path = qmk.path.under_qmk_firmware(*args)
+    def _impl_find_keymap_from_dir(relative_path):
+        if relative_path and len(relative_path.parts) > 1:
+            # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name.
+            if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts:
+                current_path = Path('/'.join(relative_path.parts[1:]))  # Strip 'keyboards' from the front
 
-    if relative_path and len(relative_path.parts) > 1:
-        # If we're in `qmk_firmware/keyboards` and `keymaps` is in our path, try to find the keyboard name.
-        if relative_path.parts[0] == 'keyboards' and 'keymaps' in relative_path.parts:
-            current_path = Path('/'.join(relative_path.parts[1:]))  # Strip 'keyboards' from the front
+                if 'keymaps' in current_path.parts and current_path.name != 'keymaps':
+                    while current_path.parent.name != 'keymaps':
+                        current_path = current_path.parent
 
-            if 'keymaps' in current_path.parts and current_path.name != 'keymaps':
-                while current_path.parent.name != 'keymaps':
-                    current_path = current_path.parent
+                    return current_path.name, 'keymap_directory'
 
-                return current_path.name, 'keymap_directory'
+            # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in
+            elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path):
+                return relative_path.name, 'layouts_directory'
 
-        # If we're in `qmk_firmware/layouts` guess the name from the community keymap they're in
-        elif relative_path.parts[0] == 'layouts' and is_keymap_dir(relative_path):
-            return relative_path.name, 'layouts_directory'
+            # If we're in `qmk_firmware/users` guess the name from the userspace they're in
+            elif relative_path.parts[0] == 'users':
+                # Guess the keymap name based on which userspace they're in
+                return relative_path.parts[1], 'users_directory'
+        return None, None
 
-        # If we're in `qmk_firmware/users` guess the name from the userspace they're in
-        elif relative_path.parts[0] == 'users':
-            # Guess the keymap name based on which userspace they're in
-            return relative_path.parts[1], 'users_directory'
+    if HAS_QMK_USERSPACE:
+        name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_userspace(*args))
+        if name and source:
+            return name, source
 
-    return None, None
+    name, source = _impl_find_keymap_from_dir(qmk.path.under_qmk_firmware(*args))
+    if name and source:
+        return name, source
+
+    return (None, None)
 
 
 def keymap_completer(prefix, action, parser, parsed_args):
@@ -417,29 +427,45 @@ def locate_keymap(keyboard, keymap):
         raise KeyError('Invalid keyboard: ' + repr(keyboard))
 
     # Check the keyboard folder first, last match wins
-    checked_dirs = ''
     keymap_path = ''
 
-    for dir in keyboard_folder(keyboard).split('/'):
-        if checked_dirs:
-            checked_dirs = '/'.join((checked_dirs, dir))
-        else:
-            checked_dirs = dir
+    search_dirs = [QMK_FIRMWARE]
+    keyboard_dirs = [keyboard_folder(keyboard)]
+    if HAS_QMK_USERSPACE:
+        # When we've got userspace, check there _last_ as we want them to override anything in the main repo.
+        search_dirs.append(QMK_USERSPACE)
+        # We also want to search for any aliases as QMK's folder structure may have changed, with an alias, but the user
+        # hasn't updated their keymap location yet.
+        keyboard_dirs.extend(keyboard_aliases(keyboard))
+        keyboard_dirs = list(set(keyboard_dirs))
+
+    for search_dir in search_dirs:
+        for keyboard_dir in keyboard_dirs:
+            checked_dirs = ''
+            for dir in keyboard_dir.split('/'):
+                if checked_dirs:
+                    checked_dirs = '/'.join((checked_dirs, dir))
+                else:
+                    checked_dirs = dir
 
-        keymap_dir = Path('keyboards') / checked_dirs / 'keymaps'
+                keymap_dir = Path(search_dir) / Path('keyboards') / checked_dirs / 'keymaps'
 
-        if (keymap_dir / keymap / 'keymap.c').exists():
-            keymap_path = keymap_dir / keymap / 'keymap.c'
-        if (keymap_dir / keymap / 'keymap.json').exists():
-            keymap_path = keymap_dir / keymap / 'keymap.json'
+                if (keymap_dir / keymap / 'keymap.c').exists():
+                    keymap_path = keymap_dir / keymap / 'keymap.c'
+                if (keymap_dir / keymap / 'keymap.json').exists():
+                    keymap_path = keymap_dir / keymap / 'keymap.json'
 
-    if keymap_path:
-        return keymap_path
+        if keymap_path:
+            return keymap_path
 
     # Check community layouts as a fallback
     info = info_json(keyboard)
 
-    for community_parent in Path('layouts').glob('*/'):
+    community_parents = list(Path('layouts').glob('*/'))
+    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
+        community_parents.append(Path(QMK_USERSPACE) / "layouts")
+
+    for community_parent in community_parents:
         for layout in info.get("community_layouts", []):
             community_layout = community_parent / layout / keymap
             if community_layout.exists():
@@ -449,6 +475,16 @@ def locate_keymap(keyboard, keymap):
                     return community_layout / 'keymap.c'
 
 
+def is_keymap_target(keyboard, keymap):
+    if keymap == 'all':
+        return True
+
+    if locate_keymap(keyboard, keymap):
+        return True
+
+    return False
+
+
 def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False):
     """List the available keymaps for a keyboard.
 
@@ -473,26 +509,30 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa
     """
     names = set()
 
-    keyboards_dir = Path('keyboards')
-    kb_path = keyboards_dir / keyboard
-
     # walk up the directory tree until keyboards_dir
     # and collect all directories' name with keymap.c file in it
-    while kb_path != keyboards_dir:
-        keymaps_dir = kb_path / "keymaps"
-
-        if keymaps_dir.is_dir():
-            for keymap in keymaps_dir.iterdir():
-                if is_keymap_dir(keymap, c, json, additional_files):
-                    keymap = keymap if fullpath else keymap.name
-                    names.add(keymap)
+    for search_dir in [QMK_FIRMWARE, QMK_USERSPACE] if HAS_QMK_USERSPACE else [QMK_FIRMWARE]:
+        keyboards_dir = search_dir / Path('keyboards')
+        kb_path = keyboards_dir / keyboard
+
+        while kb_path != keyboards_dir:
+            keymaps_dir = kb_path / "keymaps"
+            if keymaps_dir.is_dir():
+                for keymap in keymaps_dir.iterdir():
+                    if is_keymap_dir(keymap, c, json, additional_files):
+                        keymap = keymap if fullpath else keymap.name
+                        names.add(keymap)
 
-        kb_path = kb_path.parent
+            kb_path = kb_path.parent
 
     # Check community layouts as a fallback
     info = info_json(keyboard)
 
-    for community_parent in Path('layouts').glob('*/'):
+    community_parents = list(Path('layouts').glob('*/'))
+    if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
+        community_parents.append(Path(QMK_USERSPACE) / "layouts")
+
+    for community_parent in community_parents:
         for layout in info.get("community_layouts", []):
             cl_path = community_parent / layout
             if cl_path.is_dir():

lib/python/qmk/path.py

@@ -5,7 +5,7 @@ import os
 import argparse
 from pathlib import Path
 
-from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE
+from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE
 from qmk.errors import NoSuchKeyboardError
 
 
@@ -28,6 +28,40 @@ def under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):
         return None
 
 
+def under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
+    """Returns a Path object representing the relative path under $QMK_USERSPACE, or None.
+    """
+    try:
+        if HAS_QMK_USERSPACE:
+            return path.relative_to(QMK_USERSPACE)
+    except ValueError:
+        pass
+    return None
+
+
+def is_under_qmk_firmware(path=Path(os.environ['ORIG_CWD'])):
+    """Returns a boolean if the input path is a child under qmk_firmware.
+    """
+    if path is None:
+        return False
+    try:
+        return Path(os.path.commonpath([Path(path), QMK_FIRMWARE])) == QMK_FIRMWARE
+    except ValueError:
+        return False
+
+
+def is_under_qmk_userspace(path=Path(os.environ['ORIG_CWD'])):
+    """Returns a boolean if the input path is a child under $QMK_USERSPACE.
+    """
+    if path is None:
+        return False
+    try:
+        if HAS_QMK_USERSPACE:
+            return Path(os.path.commonpath([Path(path), QMK_USERSPACE])) == QMK_USERSPACE
+    except ValueError:
+        return False
+
+
 def keyboard(keyboard_name):
     """Returns the path to a keyboard's directory relative to the qmk root.
     """
@@ -45,11 +79,28 @@ def keymaps(keyboard_name):
     keyboard_folder = keyboard(keyboard_name)
     found_dirs = []
 
+    if HAS_QMK_USERSPACE:
+        this_keyboard_folder = Path(QMK_USERSPACE) / keyboard_folder
+        for _ in range(MAX_KEYBOARD_SUBFOLDERS):
+            if (this_keyboard_folder / 'keymaps').exists():
+                found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
+
+            this_keyboard_folder = this_keyboard_folder.parent
+            if this_keyboard_folder.resolve() == QMK_USERSPACE.resolve():
+                break
+
+        # We don't have any relevant keymap directories in userspace, so we'll use the fully-qualified path instead.
+        if len(found_dirs) == 0:
+            found_dirs.append((QMK_USERSPACE / keyboard_folder / 'keymaps').resolve())
+
+    this_keyboard_folder = QMK_FIRMWARE / keyboard_folder
     for _ in range(MAX_KEYBOARD_SUBFOLDERS):
-        if (keyboard_folder / 'keymaps').exists():
-            found_dirs.append((keyboard_folder / 'keymaps').resolve())
+        if (this_keyboard_folder / 'keymaps').exists():
+            found_dirs.append((this_keyboard_folder / 'keymaps').resolve())
 
-        keyboard_folder = keyboard_folder.parent
+        this_keyboard_folder = this_keyboard_folder.parent
+        if this_keyboard_folder.resolve() == QMK_FIRMWARE.resolve():
+            break
 
     if len(found_dirs) > 0:
         return found_dirs

lib/python/qmk/userspace.py

@@ -0,0 +1,185 @@
+# Copyright 2023 Nick Brassel (@tzarc)
+# SPDX-License-Identifier: GPL-2.0-or-later
+from os import environ
+from pathlib import Path
+import json
+import jsonschema
+
+from milc import cli
+
+from qmk.json_schema import validate, json_load
+from qmk.json_encoders import UserspaceJSONEncoder
+
+
+def qmk_userspace_paths():
+    test_dirs = []
+
+    # If we're already in a directory with a qmk.json and a keyboards or layouts directory, interpret it as userspace
+    current_dir = Path(environ['ORIG_CWD'])
+    while len(current_dir.parts) > 1:
+        if (current_dir / 'qmk.json').is_file():
+            test_dirs.append(current_dir)
+        current_dir = current_dir.parent
+
+    # If we have a QMK_USERSPACE environment variable, use that
+    if environ.get('QMK_USERSPACE') is not None:
+        current_dir = Path(environ.get('QMK_USERSPACE'))
+        if current_dir.is_dir():
+            test_dirs.append(current_dir)
+
+    # If someone has configured a directory, use that
+    if cli.config.user.overlay_dir is not None:
+        current_dir = Path(cli.config.user.overlay_dir)
+        if current_dir.is_dir():
+            test_dirs.append(current_dir)
+
+    return test_dirs
+
+
+def qmk_userspace_validate(path):
+    # Construct a UserspaceDefs object to ensure it validates correctly
+    if (path / 'qmk.json').is_file():
+        UserspaceDefs(path / 'qmk.json')
+        return
+
+    # No qmk.json file found
+    raise FileNotFoundError('No qmk.json file found.')
+
+
+def detect_qmk_userspace():
+    # Iterate through all the detected userspace paths and return the first one that validates correctly
+    test_dirs = qmk_userspace_paths()
+    for test_dir in test_dirs:
+        try:
+            qmk_userspace_validate(test_dir)
+            return test_dir
+        except FileNotFoundError:
+            continue
+        except UserspaceValidationError:
+            continue
+    return None
+
+
+class UserspaceDefs:
+    def __init__(self, userspace_json: Path):
+        self.path = userspace_json
+        self.build_targets = []
+        json = json_load(userspace_json)
+
+        exception = UserspaceValidationError()
+        success = False
+
+        try:
+            validate(json, 'qmk.user_repo.v0')  # `qmk.json` must have a userspace_version at minimum
+        except jsonschema.ValidationError as err:
+            exception.add('qmk.user_repo.v0', err)
+            raise exception
+
+        # Iterate through each version of the schema, starting with the latest and decreasing to v1
+        try:
+            validate(json, 'qmk.user_repo.v1')
+            self.__load_v1(json)
+            success = True
+        except jsonschema.ValidationError as err:
+            exception.add('qmk.user_repo.v1', err)
+
+        if not success:
+            raise exception
+
+    def save(self):
+        target_json = {
+            "userspace_version": "1.0",  # Needs to match latest version
+            "build_targets": []
+        }
+
+        for e in self.build_targets:
+            if isinstance(e, dict):
+                target_json['build_targets'].append([e['keyboard'], e['keymap']])
+            elif isinstance(e, Path):
+                target_json['build_targets'].append(str(e.relative_to(self.path.parent)))
+
+        try:
+            # Ensure what we're writing validates against the latest version of the schema
+            validate(target_json, 'qmk.user_repo.v1')
+        except jsonschema.ValidationError as err:
+            cli.log.error(f'Could not save userspace file: {err}')
+            return False
+
+        # Only actually write out data if it changed
+        old_data = json.dumps(json.loads(self.path.read_text()), cls=UserspaceJSONEncoder, sort_keys=True)
+        new_data = json.dumps(target_json, cls=UserspaceJSONEncoder, sort_keys=True)
+        if old_data != new_data:
+            self.path.write_text(new_data)
+            cli.log.info(f'Saved userspace file to {self.path}.')
+        return True
+
+    def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
+        if json_path is not None:
+            # Assume we're adding a json filename/path
+            json_path = Path(json_path)
+            if json_path not in self.build_targets:
+                self.build_targets.append(json_path)
+                if do_print:
+                    cli.log.info(f'Added {json_path} to userspace build targets.')
+            else:
+                cli.log.info(f'{json_path} is already a userspace build target.')
+
+        elif keyboard is not None and keymap is not None:
+            # Both keyboard/keymap specified
+            e = {"keyboard": keyboard, "keymap": keymap}
+            if e not in self.build_targets:
+                self.build_targets.append(e)
+                if do_print:
+                    cli.log.info(f'Added {keyboard}:{keymap} to userspace build targets.')
+            else:
+                if do_print:
+                    cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
+
+    def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
+        if json_path is not None:
+            # Assume we're removing a json filename/path
+            json_path = Path(json_path)
+            if json_path in self.build_targets:
+                self.build_targets.remove(json_path)
+                if do_print:
+                    cli.log.info(f'Removed {json_path} from userspace build targets.')
+            else:
+                cli.log.info(f'{json_path} is not a userspace build target.')
+
+        elif keyboard is not None and keymap is not None:
+            # Both keyboard/keymap specified
+            e = {"keyboard": keyboard, "keymap": keymap}
+            if e in self.build_targets:
+                self.build_targets.remove(e)
+                if do_print:
+                    cli.log.info(f'Removed {keyboard}:{keymap} from userspace build targets.')
+            else:
+                if do_print:
+                    cli.log.info(f'{keyboard}:{keymap} is not a userspace build target.')
+
+    def __load_v1(self, json):
+        for e in json['build_targets']:
+            if isinstance(e, list) and len(e) == 2:
+                self.add_target(keyboard=e[0], keymap=e[1], do_print=False)
+            if isinstance(e, str):
+                p = self.path.parent / e
+                if p.exists() and p.suffix == '.json':
+                    self.add_target(json_path=p, do_print=False)
+
+
+class UserspaceValidationError(Exception):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.__exceptions = []
+
+    def __str__(self):
+        return self.message
+
+    @property
+    def exceptions(self):
+        return self.__exceptions
+
+    def add(self, schema, exception):
+        self.__exceptions.append((schema, exception))
+        errorlist = "\n\n".join([f"{schema}: {exception}" for schema, exception in self.__exceptions])
+        self.message = f'Could not validate against any version of the userspace schema. Errors:\n\n{errorlist}'