From bbb75ab9b3daae9f6fd0a07db3e427f69f212743 Mon Sep 17 00:00:00 2001 From: John Unland Date: Tue, 28 Mar 2023 23:23:27 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A8=20Improve=20Docker=20local=20tests?= =?UTF-8?q?=20support=20(#25583)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 42 +++++++++------------ Makefile | 21 +++++++---- buildroot/share/scripts/get_test_targets.py | 15 ++++++++ docker-compose.yml | 19 ---------- docker/Dockerfile | 23 +++++++++-- get_test_targets.py | 12 ------ 6 files changed, 66 insertions(+), 66 deletions(-) create mode 100644 buildroot/share/scripts/get_test_targets.py delete mode 100644 docker-compose.yml delete mode 100755 get_test_targets.py diff --git a/.gitignore b/.gitignore index 555bde8622..a7f8a091aa 100755 --- a/.gitignore +++ b/.gitignore @@ -28,29 +28,11 @@ mczip.h *.gen *.sublime-workspace -# # OS -# applet/ .DS_Store -# -# Misc -# -*~ -*.orig -*.rej -*.bak -*.idea -*.i -*.ii -*.swp -tags - -# -# C++ -# -# Compiled Object files +# Compiled C++ Object files *.slo *.lo *.o @@ -81,10 +63,7 @@ tags *.out *.app -# -# C -# -# Object files +# Compiled C Object files *.o *.ko *.obj @@ -144,13 +123,13 @@ vc-fileutils.settings .vscode/* !.vscode/extensions.json -#Simulation +# Simulation files imgui.ini eeprom.dat spi_flash.bin fs.img -#cmake +# CMake CMakeLists.txt src/CMakeLists.txt CMakeListsPrivate.txt @@ -171,3 +150,16 @@ __pycache__ # IOLogger logs *_log.csv + +# Misc. +*~ +*.orig +*.rej +*.bak +*.idea +*.i +*.ii +*.swp +tags +*.logs +*.bak diff --git a/Makefile b/Makefile index d0495dc7d8..2a18c0b4e8 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,16 @@ +SCRIPTS_DIR := buildroot/share/scripts +CONTAINER_RT_BIN := docker +CONTAINER_RT_OPTS := --rm -v $(PWD):/code -v platformio-cache:/root/.platformio +CONTAINER_IMAGE := marlin-dev + help: @echo "Tasks for local development:" @echo "* tests-single-ci: Run a single test from inside the CI" @echo "* tests-single-local: Run a single test locally" - @echo "* tests-single-local-docker: Run a single test locally, using docker-compose" + @echo "* tests-single-local-docker: Run a single test locally, using docker" @echo "* tests-all-local: Run all tests locally" - @echo "* tests-all-local-docker: Run all tests locally, using docker-compose" - @echo "* setup-local-docker: Setup local docker-compose" + @echo "* tests-all-local-docker: Run all tests locally, using docker" + @echo "* setup-local-docker: Build the local docker image" @echo "" @echo "Options for testing:" @echo " TEST_TARGET Set when running tests-single-*, to select the" @@ -34,19 +39,21 @@ tests-single-local: tests-single-local-docker: @if ! test -n "$(TEST_TARGET)" ; then echo "***ERROR*** Set TEST_TARGET= or use make tests-all-local-docker" ; return 1; fi - docker-compose run --rm marlin $(MAKE) tests-single-local TEST_TARGET=$(TEST_TARGET) VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) ONLY_TEST="$(ONLY_TEST)" + @if ! $(CONTAINER_RT_BIN) images -q $(CONTAINER_IMAGE) > /dev/null ; then $(MAKE) setup-local-docker ; fi + $(CONTAINER_RT_BIN) run $(CONTAINER_RT_OPTS) $(CONTAINER_IMAGE) $(MAKE) tests-single-local TEST_TARGET=$(TEST_TARGET) VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) ONLY_TEST="$(ONLY_TEST)" .PHONY: tests-single-local-docker tests-all-local: export PATH="./buildroot/bin/:./buildroot/tests/:${PATH}" \ && export VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) \ - && for TEST_TARGET in $$(./get_test_targets.py) ; do echo "Running tests for $$TEST_TARGET" ; run_tests . $$TEST_TARGET ; done + && for TEST_TARGET in $$($(SCRIPTS_DIR)/get_test_targets.py) ; do echo "Running tests for $$TEST_TARGET" ; run_tests . $$TEST_TARGET ; done .PHONY: tests-all-local tests-all-local-docker: - docker-compose run --rm marlin $(MAKE) tests-all-local VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) + @if ! $(CONTAINER_RT_BIN) images -q $(CONTAINER_IMAGE) > /dev/null ; then $(MAKE) setup-local-docker ; fi + $(CONTAINER_RT_BIN) run $(CONTAINER_RT_OPTS) $(CONTAINER_IMAGE) $(MAKE) tests-all-local VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) .PHONY: tests-all-local-docker setup-local-docker: - docker-compose build + $(CONTAINER_RT_BIN) build -t $(CONTAINER_IMAGE) -f docker/Dockerfile . .PHONY: setup-local-docker diff --git a/buildroot/share/scripts/get_test_targets.py b/buildroot/share/scripts/get_test_targets.py new file mode 100644 index 0000000000..f51951ae12 --- /dev/null +++ b/buildroot/share/scripts/get_test_targets.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +""" +Extract the builds used in Github CI, so that we can run them locally +""" +import yaml + +# Set the yaml file to parse +yaml_file = '.github/workflows/test-builds.yml' + +# Parse the yaml file, and load it into a dictionary (github_configuration) +with open(yaml_file) as f: + github_configuration = yaml.safe_load(f) + +# Print out the test platforms +print(' '.join(github_configuration['jobs']['test_builds']['strategy']['matrix']['test-platform'])) diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index a85e0f8315..0000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,19 +0,0 @@ -version: "3.8" -services: - # The main image: this doesn't run any particular command, but is mainly used - # for running tests locally - marlin: - image: marlin-dev - build: - dockerfile: Dockerfile - context: docker - working_dir: /code - volumes: - - .:/code - - platformio-cache:/root/.platformio - -volumes: - # This volume holds installed libraries for PlatformIO. If this is deleted you - # will have to download all the dependencies again, which can be a very slow - # process - platformio-cache: diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d32f9c637..c01ca76ebb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,24 @@ FROM python:3.9.0-buster -RUN pip install -U platformio +# Disable warnings about not having a TTY +ARG DEBIAN_FRONTEND=noninteractive + +# Disable debconf warnings +ARG DEBCONF_NOWARNINGS="yes" + +# Upgrade pip +RUN pip install --upgrade pip + +# Install platformio toolchain / framework and pyyaml +RUN pip install -U platformio PyYaml + +# Upgrade platformio using development version / branch RUN pio upgrade --dev -# To get the test platforms -RUN pip install PyYaml + +# Set working directory +WORKDIR /code + +# Set volumes / mount points that we are using +VOLUME /code /root/.platformio + #ENV PATH /code/buildroot/bin/:/code/buildroot/tests/:${PATH} diff --git a/get_test_targets.py b/get_test_targets.py deleted file mode 100755 index a38e3a594a..0000000000 --- a/get_test_targets.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python -""" -Extract the builds used in Github CI, so that we can run them locally -""" -import yaml - - -with open('.github/workflows/test-builds.yml') as f: - github_configuration = yaml.safe_load(f) -test_platforms = github_configuration\ - ['jobs']['test_builds']['strategy']['matrix']['test-platform'] -print(' '.join(test_platforms))