From ceb6b1affd29be396396e36f9ab64617b0187efc Mon Sep 17 00:00:00 2001 From: dam1n19 <dam1n19@soton.ac.uk> Date: Fri, 30 Jun 2023 20:56:40 +0100 Subject: [PATCH] Added Regression Flow to Makefile --- makefile | 129 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 56 deletions(-) diff --git a/makefile b/makefile index 382daf5..faa9852 100644 --- a/makefile +++ b/makefile @@ -60,7 +60,7 @@ TESTNAME = TEST_LIST_FILE ?= $(TESTCODES_DIR)/software_list.txt TEST_LIST_FILE += $(PROJ_SW_DIR)/software_list.txt -TEST_LIST = $(shell cat $(TEST_LIST_FILE) | while read line || [ -n "$$line" ]; do echo $$line; done) +TEST_LIST = $(shell cat $(TEST_LIST_FILE) | while read line || [ -n "$$line" ]; do echo $$line; done) # Default to DS-5 tool-chain TOOL_CHAIN = ds5 @@ -163,6 +163,49 @@ DEBUGTESTER = debugtester # You can append additional make options for testcode here SW_MAKE_OPTIONS = CPU_PRODUCT=$(CPU_PRODUCT) TOOL_CHAIN=$(TOOL_CHAIN) +$(eval REGRESSION_NAME =$(shell date +%Y_%m_%d_%H_%M_%S)) +$(eval REGRESSION_DIR = $(SIM_TOP_DIR)/regression_$(REGRESSION_NAME)) + +# Create a Template for Regression simulations + +# Create List of Make Targets +TEST_LIST_MTI = $(shell cat $(TEST_LIST_FILE) | while read line || [ -n "$$line" ]; do echo mti_$$line; done) +TEST_LIST_XM = $(shell cat $(TEST_LIST_FILE) | while read line || [ -n "$$line" ]; do echo mti_$$line; done) +TEST_LIST_VCS = $(shell cat $(TEST_LIST_FILE) | while read line || [ -n "$$line" ]; do echo mti_$$line; done) + +# Create templates for Make Targets +define mti_template +.PHONY: $(1) +mti_$(1): + @echo Running test $(1) + @$(MAKE) -j1 run_mti TESTNAME=$(1) > $$(REGRESSION)/$(1).log + @if cat $$(REGRESSION)/$(1).log | grep -i "Test Passed"; then echo "$(1): PASSED" >> $$(REGRESSION)/RESULTS.log; else echo "$(1): FAILED" >> $$(REGRESSION)/RESULTS.log; fi > /dev/null +endef + +define xm_template +.PHONY: $(1) +xm_$(1): + @echo Running test $(1) + @$(MAKE) -j1 run_xm TESTNAME=$(1) > $$(REGRESSION)/$(1).log + @if cat $$(REGRESSION)/$(1).log | grep -i "Test Passed"; then echo "$(1): PASSED" >> $$(REGRESSION)/RESULTS.log; else echo "$(1): FAILED" >> $$(REGRESSION)/RESULTS.log; fi > /dev/null +endef + +define vcs_template +.PHONY: $(1) +vcs_$(1): + @echo Running test $(1) + @$(MAKE) -j1 run_vcs TESTNAME=$(1) > $$(REGRESSION)/$(1).log + @if cat $$(REGRESSION)/$(1).log | grep -i "Test Passed"; then echo "$(1): PASSED" >> $$(REGRESSION)/RESULTS.log; else echo "$(1): FAILED" >> $$(REGRESSION)/RESULTS.log; fi > /dev/null +endef + +# Generate Targets from Templates using Test List +$(foreach TESTNAME,$(TEST_LIST), $(eval $(call mti_template,$(TESTNAME)))) +$(foreach TESTNAME,$(TEST_LIST), $(eval $(call xm_template,$(TESTNAME)))) +$(foreach TESTNAME,$(TEST_LIST), $(eval $(call vcs_template,$(TESTNAME)))) + +# Create a List of PHONY Targets +.PHONY: run_$(SIMULATOR) sim_$(SIMULATOR) all_$(SIMULATOR) test_$(SIMULATOR) + # ------- Simulator redirect ----------- compile : compile_$(SIMULATOR) @@ -195,24 +238,6 @@ run_vcs : code sim_vcs : code ./simv -gui +vcs+lic+wait +vcs+flush+log & -# Compile RTL, and run all tests in batch mode -all_vcs : compile_vcs bootrom debugtester - @if [ ! -d logs ] ; then \ - mkdir logs; \ - fi - @echo quit > quit.do - @echo Run tests ... - for thistest in $(TEST_LIST) ; do \ - echo $$thistest ; \ - make testcode TESTNAME=$$thistest ;\ - if [ -e image.hex ] ; then \ - ./simv +vcs+lic+wait +vcs+flush+log < quit.do | tee logs/run_$$thistest.log ;\ - else \ - echo Cannot read image.hex ;\ - exit 1; \ - fi ; \ - done - # ------- XM ----------- @@ -247,27 +272,6 @@ sim_xm : code compile_xm cd $(SIM_DIR); xmsim -gui $(XMSIM_OPTIONS) # @make verify -# Compile RTL, and run all tests in batch mode -all_xm : compile_xm bootrom debugtester - @if [ ! -d $(SIM_DIR)/logs ] ; then \ - mkdir $(SIM_DIR)/logs; \ - fi - @echo run > $(SIM_DIR)run.tcl.tmp - @echo exit >> $(SIM_DIR)run.tcl.tmp - @mv $(SIM_DIR)run.tcl.tmp $(SIM_DIR)run.tcl - @echo Run tests ... - for thistest in $(TEST_LIST) ; do \ - echo $$thistest ; \ - make testcode TESTNAME=$$thistest ;\ - if [ -e image.hex ] ; then \ - cd $(SIM_DIR); xmsim $(XMSIM_OPTIONS) -input run.tcl | tee logs/run_$$thistest.log ;\ - else \ - echo Cannot read image.hex ;\ - exit 1; \ - fi ; \ - done - @make verify - # ------- MTI ----------- # Compile RTL @@ -292,6 +296,9 @@ run_mti : code compile_mti @mv $(SIM_DIR)/run.tcl.tmp $(SIM_DIR)/run.tcl cd $(SIM_DIR); vsim $(MTI_RUN_OPTIONS) -c $(TB_TOP) -do run.tcl | tee logs/run_$(TESTNAME).log ; +test_mti: + @echo $(TESTNAME) + # Run simulation in interactive mode sim_mti : code compile_mti cd $(SIM_DIR); vsim $(MTI_RUN_OPTIONS) -gui $(TB_TOP) & @@ -300,23 +307,33 @@ sim_mti : code compile_mti lib_mti : vlib work -# Compile RTL, and run all tests in batch mode -all_mti : bootrom compile_mti debugtester - @if [ ! -d logs ] ; then \ - mkdir logs; \ - fi - @echo Run tests ... - for thistest in $(TEST_LIST) ; do \ - echo $$thistest ; \ - make testcode TESTNAME=$$thistest ;\ - if [ -e image.hex ] ; then \ - vsim $(MTI_OPTIONS) -c tb_cmsdk_mcu -do "radix hex;run -all;quit -f" | tee logs/run_$$thistest.log ;\ - else \ - echo Cannot read image.hex ;\ - exit 1; \ - fi ; \ - done +# Create Targets with dependencies for all tests for simulator +all_mti : $(TEST_LIST_MTI) +all_vcs : $(TEST_LIST_VCS) +all_xm : $(TEST_LIST_XM) +regression_results: + @echo -------------------------------------------- + @echo Results: + @echo -------------------------------------------- + @python $(SOCLABS_SOCTOOLS_FLOW_DIR)/tools/regression_result.py $(REGRESSION)/RESULTS.log + +# Create a Target for Running Simulation Regressions +regression_$(SIMULATOR): compile_$(SIMULATOR) bootrom debugtester + @mkdir -p $(REGRESSION_DIR) + @echo + @echo + @echo + @echo + @echo -------------------------------------------- + @echo Running Regression: $(REGRESSION_NAME) + @echo -------------------------------------------- + @echo Simulator: $(SIMULATOR) + @echo -------------------------------------------- + @make -j4 all_$(SIMULATOR) REGRESSION=$(REGRESSION_DIR) + @make -j1 regression_results REGRESSION=$(REGRESSION_DIR) + + # ------- Software ----------- # code target is used by other simulation targets (e.g. run_mti, sim_mti) -- GitLab