Illusion-of-Reasoning/Makefile at master · humans-and-machines/Illusion-of-Reasoning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
.PHONY: style quality

# make sure to test the local checkout in scripts and not the pre-installed one (don't use quotes!)
export PYTHONPATH = src

check_dirs := src tests


# dev dependencies
install:
	mkdir -p $(PWD)/.pip_cache $(PWD)/.tmp $(PWD)/.hf_cache $(PWD)/.conda_pkgs && \
	PIP_CACHE_DIR="$(PWD)/.pip_cache" TMPDIR="$(PWD)/.tmp" HF_HOME="$(PWD)/.hf_cache" CONDA_PKGS_DIRS="$(PWD)/.conda_pkgs" \
	. $$(conda info --base)/etc/profile.d/conda.sh && \
		(conda env create -f configs/environment.yml -p $(PWD)/openr1 || conda env update -f configs/environment.yml -p $(PWD)/openr1) && \
		conda activate $(PWD)/openr1 && \
		PIP_CACHE_DIR="$(PWD)/.pip_cache" TMPDIR="$(PWD)/.tmp" HF_HOME="$(PWD)/.hf_cache" CONDA_PKGS_DIRS="$(PWD)/.conda_pkgs" \
		pip install --upgrade pip && \
		PIP_CACHE_DIR="$(PWD)/.pip_cache" TMPDIR="$(PWD)/.tmp" HF_HOME="$(PWD)/.hf_cache" CONDA_PKGS_DIRS="$(PWD)/.conda_pkgs" \
		GIT_LFS_SKIP_SMUDGE=1 pip install -e ".[dev]"

style:
	ruff format --line-length 119 --target-version py310 $(check_dirs) setup.py
	isort $(check_dirs) setup.py

quality:
	ruff check --line-length 119 --target-version py310 $(check_dirs) setup.py
	isort --check-only $(check_dirs) setup.py
	flake8 --max-line-length 119 $(check_dirs) setup.py

test:
	pytest -sv --ignore=tests/slow/ tests/

slow_test:
	pytest -sv -vv tests/slow/

# Evaluation

evaluate:
	$(eval PARALLEL_ARGS := $(if $(PARALLEL),$(shell \
		if [ "$(PARALLEL)" = "data" ]; then \
			echo "data_parallel_size=$(NUM_GPUS)"; \
		elif [ "$(PARALLEL)" = "tensor" ]; then \
			echo "tensor_parallel_size=$(NUM_GPUS)"; \
		fi \
	),))
	$(if $(filter tensor,$(PARALLEL)),export VLLM_WORKER_MULTIPROC_METHOD=spawn &&,) \
	MODEL_ARGS="pretrained=$(MODEL),dtype=bfloat16,$(PARALLEL_ARGS),max_model_length=32768,gpu_memory_utilization=0.8,generation_parameters={max_new_tokens:32768,temperature:0.6,top_p:0.95}" && \
	if [ "$(TASK)" = "lcb" ]; then \
		lighteval vllm $$MODEL_ARGS "extended|lcb:codegeneration|0|0" \
			--use-chat-template \
			--output-dir data/evals/$(MODEL); \
	else \
		lighteval vllm $$MODEL_ARGS "lighteval|$(TASK)|0|0" \
			--use-chat-template \
			--output-dir data/evals/$(MODEL); \
	fi