Experiments

ID	Engine	Model	Prompt	Evaluations	Pass@1	Compile@1
104	mlx	ipetrukha/CodeQwen1.5-7B-4bit	completion	1580	0.37	0.62
51	mlx	ipetrukha/CodeQwen1.5-7B-Chat-4bit	default	1580	0.67	0.90
52	mlx	ipetrukha/Nxcode-CQ-7B-orpo-4bit	default	1580	0.66	0.90
87	mlx	mlx-community/CodeLlama-7b-Instruct-hf-4bit-MLX	instruct	1580	0.30	0.76
88	mlx	mlx-community/CodeLlama-13b-Instruct-hf-4bit-MLX	instruct	1580	0.36	0.86
101	mlx	mlx-community/CodeLlama-34b-Instruct-hf-4bit	instruct	1580	0.44	0.91
48	mlx	mlx-community/Codestral-22B-v0.1-4bit	instruct	1580	0.73	0.93
27	mlx	mlx-community/codegemma-2b-4bit	completion	1580	0.26	0.77
77	mlx	mlx-community/codegemma-7b-4bit	completion	1580	0.41	0.88
28	mlx	mlx-community/codegemma-7b-it-4bit	default	1580	0.41	0.80
43	mlx	mlx-community/deepseek-coder-1.3b-instruct-mlx	default	1580	0.36	0.68
46	mlx	mlx-community/deepseek-coder-6.7b-instruct-hf-4bit-mlx	default	1580	0.50	0.81
47	mlx	mlx-community/deepseek-coder-33b-instruct-hf-4bit-mlx	default	1580	0.63	0.83
32	mlx	mlx-community/granite-3b-code-instruct-4bit	default	1580	0.23	0.59
38	mlx	mlx-community/granite-8b-code-instruct-4bit	default	1580	0.33	0.63
39	mlx	mlx-community/granite-20b-code-instruct-4bit	default	1580	0.30	0.55
40	mlx	mlx-community/granite-34b-code-instruct-4bit	default	1580	0.39	0.67
41	mlx	mlx-community/stable-code-3b-4bit	completion	1580	0.00	0.00
42	mlx	mlx-community/stable-code-instruct-3b-4bit	default	1580	0.01	0.01
29	mlx	mlx-community/starcoder2-3b-4bit	completion	1580	0.22	0.69
30	mlx	mlx-community/starcoder2-7b-4bit	completion	1580	0.25	0.63
31	mlx	mlx-community/starcoder2-15b-4bit	completion	1580	0.24	0.50
143	openai	gpt-3.5-turbo	default	1580	0.65	0.96
139	openai	gpt-4	default	1580	0.80	0.98
140	openai	gpt-4-turbo	default	1580	0.84	0.97
141	openai	gpt-4o	default	1580	0.84	0.98
142	openai	gpt-4o-mini	default	1580	0.80	0.98
203	transformers	01-ai/Yi-Coder-1.5B	completion	1580	0.13	0.35
204	transformers	01-ai/Yi-Coder-1.5B-Chat	default	1580	0.38	0.69
205	transformers	01-ai/Yi-Coder-9B	completion	500	0.41	0.66
206	transformers	01-ai/Yi-Coder-9B-Chat	default	1580	0.68	0.87
137	transformers	NTQAI/Nxcode-CQ-7B-orpo	default	1580	0.68	0.91
130	transformers	Qwen/CodeQwen1.5-7B	completion	1580	0.50	0.85
131	transformers	Qwen/CodeQwen1.5-7B-Chat	default	1580	0.68	0.90
207	transformers	Qwen/Qwen2.5-Coder-1.5B	completion	1580	0.45	0.78
209	transformers	Qwen/Qwen2.5-Coder-1.5B-Instruct	default	1580	0.33	0.63
208	transformers	Qwen/Qwen2.5-Coder-7B	completion	1580	0.61	0.81
210	transformers	Qwen/Qwen2.5-Coder-7B-Instruct	default	1580	0.69	0.89
162	transformers	THUDM/codegeex2-6b	completion	1580	0.19	0.65
132	transformers	THUDM/codegeex4-all-9b	instruct	1580	0.64	0.90
164	transformers	WisdomShell/CodeShell-7B	completion	1580	0.17	0.52
166	transformers	WisdomShell/CodeShell-7B-Chat	instruct	1580	0.17	0.54
125	transformers	bigcode/starcoder2-3b	completion	1580	0.24	0.65
126	transformers	bigcode/starcoder2-7b	completion	1580	0.27	0.66
127	transformers	bigcode/starcoder2-15b	completion	1580	0.25	0.53
211	transformers	bigcode/starcoder2-15b-instruct-v0.1	instruct	1580	0.47	0.79
124	transformers	deepseek-ai/deepseek-coder-1.3b-instruct	default	1580	0.37	0.68
123	transformers	deepseek-ai/deepseek-coder-6.7b-instruct	default	1580	0.53	0.82
135	transformers	deepseek-ai/deepseek-coder-33b-instruct	default	1580	0.66	0.85
115	transformers	google/codegemma-1.1-2b	completion	1580	0.05	0.18
117	transformers	google/codegemma-1.1-7b-it	default	1580	0.44	0.80
107	transformers	google/codegemma-2b	completion	1580	0.27	0.80
113	transformers	google/codegemma-2b	completion	1580	0.28	0.80
114	transformers	google/codegemma-7b	completion	1580	0.37	0.83
116	transformers	google/codegemma-7b-it	default	1580	0.43	0.81
121	transformers	ibm-granite/granite-3b-code-instruct	default	1580	0.25	0.57
122	transformers	ibm-granite/granite-8b-code-instruct	default	1580	0.35	0.62
133	transformers	ibm-granite/granite-20b-code-instruct	default	1580	0.31	0.59
134	transformers	ibm-granite/granite-34b-code-instruct	default	1580	0.39	0.69
160	transformers	m-a-p/OpenCodeInterpreter-DS-6.7B	completion	1580	0.52	0.81
150	transformers	m-a-p/OpenCodeInterpreter-DS-33B	instruct	1580	0.59	0.84
118	transformers	meta-llama/CodeLlama-7b-Instruct-hf	default	1580	0.31	0.71
144	transformers	meta-llama/CodeLlama-7b-Python-hf	completion	1580	0.28	0.78
119	transformers	meta-llama/CodeLlama-13b-Instruct-hf	default	1580	0.36	0.74
146	transformers	meta-llama/CodeLlama-13b-Python-hf	instruct	1580	0.36	0.84
120	transformers	meta-llama/CodeLlama-34b-Instruct-hf	default	1580	0.35	0.74
138	transformers	meta-llama/CodeLlama-70b-Instruct-hf	instruct	1580	0.56	0.89
136	transformers	mistralai/Codestral-22B-v0.1	default	1580	0.73	0.94
78	transformers	stabilityai/stable-code-3b	completion	1580	0.14	0.39
128	transformers	stabilityai/stable-code-3b	completion	1580	0.14	0.41
79	transformers	stabilityai/stable-code-instruct-3b	default	1580	0.32	0.58
129	transformers	stabilityai/stable-code-instruct-3b	default	1580	0.31	0.57