sBubshait/ResBench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: fixes and configuration to path

Browse Source
This commit is contained in:
Saleh Bubshait
2025-12-17 18:16:44 +00:00
parent a4a9644f9e
commit abfd851c59
4 changed files with 130 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
functional_correctness.py
View File

@@ -10,8 +10,13 @@ TEMP_VERILOG_FILE = "temp.v"
TEMP_TESTBENCH_FILE = "testbench.v"
TCL_SCRIPT_FILE = "run_testbench.tcl"
def write_tcl():
# Generate the TCL script for Vivado
# --- CONFIGURATION ---
# Default fallback path if env var is missing
DEFAULT_VIVADO_PATH = "/tools/Xilinx/Vivado/2023.1/bin"
# FIX 1: Add 'top_module' as an argument so the function can use it
def write_tcl(top_module):
# Generate the TCL script for Vivado
tcl_commands = f"""
create_project temp_project ./temp_project -force -part xc7z020clg400-1
set_property source_mgmt_mode All [current_project]
@@ -33,14 +38,15 @@ def extract_top_module_name(testbench_file):
for line in file:
match = re.search(r'\s*module\s+(\w+)\s*;', line)
if match:
print(match.group(1))
return match.group(1) # Extract module name
return None # Return None if no module found
def run_functional_correctness():
# Load JSON files
if not os.path.exists(SOLUTIONS_FILE):
print(f"Error: {SOLUTIONS_FILE} not found.")
return
with open(SOLUTIONS_FILE, "r", encoding="utf-8") as file:
solutions_data = json.load(file)
@@ -56,34 +62,31 @@ def run_functional_correctness():
if module_name and testbench_code:
module_testbenches[module_name] = testbench_code
# print(module_testbenches.keys())
# Get Vivado path from environment variable
vivado_path = os.environ.get("vivado")
if not vivado_path:
raise EnvironmentError("Vivado environment variable not set.")
vivado_path = os.path.join(vivado_path, "vivado.bat")
# FIX 2: Handle Linux Path correctly (No .bat)
vivado_path_env = os.environ.get("vivado")
if vivado_path_env:
vivado_bin = os.path.join(vivado_path_env, "vivado")
elif os.path.exists(os.path.join(DEFAULT_VIVADO_PATH, "vivado")):
vivado_bin = os.path.join(DEFAULT_VIVADO_PATH, "vivado")
else:
vivado_bin = "vivado" # Hope it's in PATH
print(vivado_bin)
# Iterate over solutions and test them
for model, categories in solutions_data.items():
for category, modules in categories.items():
for module_entry in modules:
module_name = module_entry["module"]
# print(module_name)
# print(module_name in module_testbenches.keys())
if module_name not in module_testbenches:
print(f"Skipping {module_name}: No testbench found.")
continue
testbench_code = module_testbenches[module_name]
solutions = module_entry["solutions"]
# Iterate over all solutions
for solution_entry in solutions:
for idx, solution_entry in enumerate(solutions):
verilog_code = solution_entry["solution"]
# Write the Verilog design to a file
@@ -101,31 +104,34 @@ def run_functional_correctness():
solution_entry["pass"] = "Error: Could not extract top module."
continue
print(f"Testing module: {module_name} (Top Module: {top_module})")
print(f"Testing {module_name} (Solution {idx+1})...")
write_tcl()
# FIX 3: Pass the variable to the function
write_tcl(top_module)
# Run Vivado in batch mode
print(f"Running Vivado simulation for {module_name}...")
process = subprocess.run([vivado_path, "-mode", "batch", "-source", TCL_SCRIPT_FILE], capture_output=True, text=True)
try:
process = subprocess.run([vivado_bin, "-mode", "batch", "-source", TCL_SCRIPT_FILE], capture_output=True, text=True)
output_log = process.stdout + "\n" + process.stderr
except FileNotFoundError:
print("CRITICAL ERROR: Vivado executable not found. Check path.")
return
# Capture output logs
output_log = process.stdout + "\n" + process.stderr
print(output_log)
test_passed = "All tests passed" in output_log
# Determine pass/fail status
if test_passed:
solution_entry["pass"] = "true"
print(f" ✅ PASS")
else:
# Extract relevant error messages
error_lines = "\n".join(line for line in output_log.split("\n") if "error" or "fail" in line.lower())
solution_entry["pass"] = error_lines if error_lines else "Test failed somehow"
solution_entry["pass"] = "false" # Keep it simple for now
print(f" ❌ FAIL")
print(f"Test result for {module_name}: {'PASS' if test_passed else 'FAIL'}")
# Save results after testing each module
# Save results incrementally
with open(SOLUTIONS_FILE, "w", encoding="utf-8") as file:
json.dump(solutions_data, file, indent=4)
print("All tests completed.")
if __name__ == "__main__":
run_functional_correctness()

6
problems.json
View File

@@ -5,7 +5,7 @@
"Problem": "Implement a Verilog module that computes the parity of an 8-bit input vector. The output should be 1 if the number of '1's in the input is odd, and 0 otherwise.",
"Module header": "module parity_8bit (\n input [7:0] in,\n output out\n);",
"Testbench": "`timescale 1ns / 1ps\n\nmodule parity_8bit_tb;\n\n reg [7:0] test_in; // Input signal\n wire test_out; // Output signal\n reg expected_out; // Expected output for verification\n integer errors = 0; // Error counter\n\n // Instantiate the module under test\n parity_8bit uut (\n .in(test_in),\n .out(test_out)\n );\n\n integer i;\n reg [7:0] test_vectors[0:4]; // Test cases\n reg expected_values[0:4]; // Expected results\n\n initial begin\n // Define test cases and expected results\n test_vectors[0] = 8'b00000000; expected_values[0] = 0; // Even parity (0 ones)\n test_vectors[1] = 8'b11111111; expected_values[1] = 0; // Even parity (8 ones)\n test_vectors[2] = 8'b00000001; expected_values[2] = 1; // Odd parity (1 one)\n test_vectors[3] = 8'b10000000; expected_values[3] = 1; // Odd parity (1 one)\n test_vectors[4] = 8'b01010101; expected_values[4] = 0; // Even parity (4 ones)\n\n $display(\"==========Testbench results==========\");\n $display(\"=====================================\");\n $display(\" Test Input | Expected | Output | Pass/Fail \");\n $display(\"-------------------------------------\");\n\n for (i = 0; i < 5; i = i + 1) begin\n test_in = test_vectors[i];\n expected_out = expected_values[i];\n #10; // Wait for the output to stabilize\n\n if (test_out === expected_out) begin\n $display(\" %b | %b | %b | PASS\", test_in, expected_out, test_out);\n end else begin\n $display(\" %b | %b | %b | FAIL\", test_in, expected_out, test_out);\n errors = errors + 1;\n end\n end\n\n $display(\"=====================================\");\n if (errors == 0)\n $display(\"All tests passed\");\n else\n $display(\"Some tests failed\");\n\n $finish;\n end\n\nendmodule\n"
},
},
{
"module": "mux4to1",
"Problem": "Design a Verilog module that implements a 4-to-1 multiplexer using only basic logic gates (AND, OR, NOT).",
@@ -186,8 +186,8 @@
},
{
"module": "pipelined_max_finder",
"Problem": "Implement a pipelined module to find the maximum value in a stream of 8-bit integers over 4 clock cycles, so it takes four 8-bit inputs. Use a 2-stage pipeline for comparison and selection.",
"Module header": "module pipelined_max_finder (\n input clk, rst,\n input [7:0] x0,x1,x2,x3,\n output reg [7:0] max_value\n);",
"Problem": "Implement a pipelined module to find the maximum value in a stream of 8-bit integers over 4 clock cycles, so it takes four 8-bit inputs. Use a 2-stage pipeline for comparison and selection.",
"Module header": "module pipelined_max_finder (\n input clk, rst,\n input [7:0] x0,x1,x2,x3,\n output reg [7:0] max_value\n);",
"Testbench": "`timescale 1ns/1ps\nmodule pipelined_max_finder_tb;\n\n reg clk;\n reg rst;\n reg [7:0] x0, x1, x2, x3;\n wire [7:0] max_value;\n\n // Instantiate the Unit Under Test (UUT)\n pipelined_max_finder uut (\n .clk(clk),\n .rst(rst),\n .x0(x0),\n .x1(x1),\n .x2(x2),\n .x3(x3),\n .max_value(max_value)\n );\n\n // Clock generation: period = 10ns\n initial clk = 0;\n always #5 clk = ~clk;\n\n // Test case arrays (4 test cases)\n reg [7:0] test_x0 [0:3];\n reg [7:0] test_x1 [0:3];\n reg [7:0] test_x2 [0:3];\n reg [7:0] test_x3 [0:3];\n reg [7:0] expected [0:3];\n\n integer i;\n integer errors;\n\n initial begin\n errors = 0;\n // Initialize test vectors.\n // Test Case 0: {10, 20, 5, 15} -> Expected max = 20\n test_x0[0] = 8'd10; test_x1[0] = 8'd20; test_x2[0] = 8'd5; test_x3[0] = 8'd15; expected[0] = 8'd200;\n // Test Case 1: {100, 50, 200, 150} -> Expected max = 200\n test_x0[1] = 8'd100; test_x1[1] = 8'd50; test_x2[1] = 8'd200; test_x3[1] = 8'd150; expected[1] = 8'd0;\n // Test Case 2: {0, 0, 0, 0} -> Expected max = 0\n test_x0[2] = 8'd0; test_x1[2] = 8'd0; test_x2[2] = 8'd0; test_x3[2] = 8'd0; expected[2] = 8'd255;\n // Test Case 3: {255, 100, 200, 250} -> Expected max = 255\n test_x0[3] = 8'd255; test_x1[3] = 8'd100; test_x2[3] = 8'd200; test_x3[3] = 8'd250; expected[3] = 8'd255;\n\n // Apply reset.\n rst = 1;\n x0 = 8'd0; x1 = 8'd0; x2 = 8'd0; x3 = 8'd0;\n #12;\n rst = 0;\n\n // Apply the test vectors on successive clock cycles.\n for (i = 0; i < 4; i = i + 1) begin\n @(negedge clk);\n x0 <= test_x0[i];\n x1 <= test_x1[i];\n x2 <= test_x2[i];\n x3 <= test_x3[i];\n end\n\n // After applying test vectors, drive zeros to flush the pipeline.\n x0 = 8'd0; x1 = 8'd0; x2 = 8'd0; x3 = 8'd0;\n\n // Wait for 4 clock cycles to capture the outputs corresponding to the test cases.\n $display(\"==========Testbench results==========\");\n $display(\"=====================================\");\n $display(\" Test Input | Expected | Output | Pass/Fail \");\n $display(\"-------------------------------------\");\n for (i = 0; i < 4; i = i + 1) begin\n @(negedge clk);\n if (max_value === expected[i]) begin\n $display(\" %3d, %3d, %3d, %3d | %3d | %3d | PASS\", \n test_x0[i], test_x1[i], test_x2[i], test_x3[i], expected[i], max_value);\n end else begin\n $display(\" %3d, %3d, %3d, %3d | %3d | %3d | FAIL\", \n test_x0[i], test_x1[i], test_x2[i], test_x3[i], expected[i], max_value);\n errors = errors + 1;\n end\n end\n $display(\"=====================================\");\n if (errors == 0)\n $display(\"All tests passed\");\n else\n $display(\"Some tests failed\");\n $display(\"=====================================\");\n $finish;\n end\n\nendmodule"
},
{

144
resource_usage.py
View File

@@ -3,6 +3,10 @@ import subprocess
import os
import re
# --- CONFIGURATION ---
# If your environment variable isn't set, this default path will be used.
DEFAULT_VIVADO_PATH = "/tools/Xilinx/Vivado/2023.1/bin"
def extract_module_name(verilog_code):
"""
Extract the module name from the Verilog code.
@@ -62,10 +66,6 @@ def extract_primitives_section(lines):
def parse_primitives_section(lines):
"""
Parses the primitives section lines to accumulate resource usage.
Returns a dictionary with keys: LUT, FF, DSP, BRAM, IO.
In this example:
- For LUT: sums up any primitive whose name starts with "LUT" (e.g., LUT2, LUT3, ...)
- For IO: sums the usage of IBUF and OBUF.
"""
resources = {"LUT": 0, "FF": 0, "DSP": 0, "BRAM": 0, "IO": 0}
for line in lines:
@@ -85,110 +85,140 @@ def parse_primitives_section(lines):
resources["LUT"] += used
if ref_name in ("IBUF", "OBUF"):
resources["IO"] += used
# (Add additional processing for FF, DSP, BRAM if necessary.)
return resources
def run_synthesis(solution_code):
"""
Writes the given Verilog solution to a temporary file,
creates a Tcl script for Vivado to run synthesis and generate a utilization report,
runs Vivado in batch mode, and parses the resource usage report.
Returns a dictionary with keys "optimized" and "primitives" containing resource usage.
Runs Vivado synthesis on the provided code.
"""
# Write the Verilog code to a temporary file.
# 1. Write the Verilog code to a temporary file.
verilog_file = "temp.v"
with open(verilog_file, "w") as f:
f.write(solution_code)
# Extract the module name from the solution code.
# 2. Extract the module name
top_module = extract_module_name(solution_code)
print(top_module)
if top_module is None:
print("Could not extract module name; using 'temp_top' as a default.")
top_module = "temp_top"
vivado_project = "temp_project"
tcl_script = "synthesis_script.tcl"
# Get the Vivado installation path from the environment variable.
# 3. Resolve Vivado Path (Linux Fix)
vivado_path_env = os.environ.get("vivado")
if vivado_path_env is None:
print("Error: 'vivado' environment variable is not set.")
return None
vivado_path = os.path.join(vivado_path_env, "vivado.bat")
# If env var is missing, try the default path, or search system path
if vivado_path_env:
vivado_bin = os.path.join(vivado_path_env, "vivado") # NO .bat
elif os.path.exists(os.path.join(DEFAULT_VIVADO_PATH, "vivado")):
vivado_bin = os.path.join(DEFAULT_VIVADO_PATH, "vivado")
else:
# Last resort: assume it's in the system PATH
vivado_bin = "vivado"
# Create the Vivado Tcl script.
tcl_script = "synthesis_script.tcl"
vivado_project = "temp_project"
# 4. Create the Vivado Tcl script.
tcl_commands = f"""
create_project {vivado_project} -force -part xc7z020clg400-1
add_files {verilog_file}
set_property top {top_module} [current_fileset]
# Run synthesis only (no simulation)
synth_design -top {top_module}
synth_design -top {top_module} -part xc7z020clg400-1
# Generate resource utilization report
report_utilization -file resource_usage.rpt
quit
exit
"""
with open(tcl_script, "w") as file:
file.write(tcl_commands)
# Run Vivado in batch mode using the generated Tcl script.
# 5. Run Vivado
print(f"Starting Synthesis for {top_module} using {vivado_bin}...")
try:
# Added stdout=PIPE so we can see the output if it crashes
result = subprocess.run(
[vivado_path, "-mode", "batch", "-source", tcl_script],
[vivado_bin, "-mode", "batch", "-source", tcl_script],
capture_output=True, text=True, check=True
)
except subprocess.CalledProcessError as e:
print("Synthesis failed:", e)
print(f"❌ Synthesis CRASHED for {top_module}")
print("--- Error Log ---")
print(e.stderr) # Print the actual error from Vivado
print("-----------------")
return None
print(result.stdout)
# Check for the success message in the output.
except FileNotFoundError:
print(f"❌ CRITICAL ERROR: Could not find Vivado executable at: {vivado_bin}")
return None
# 6. Check results
if "Finished Writing Synthesis Report" in result.stdout:
# Read the resource utilization report.
with open("resource_usage.rpt", "r") as f:
report_lines = f.readlines()
optimized_resources = parse_optimized(report_lines)
primitives_section = extract_primitives_section(report_lines)
primitives_resources = (parse_primitives_section(primitives_section)
if primitives_section else {})
return {"optimized": optimized_resources, "primitives": primitives_resources}
print(f"✅ Synthesis Success: {top_module}")
if os.path.exists("resource_usage.rpt"):
with open("resource_usage.rpt", "r") as f:
report_lines = f.readlines()
optimized = parse_optimized(report_lines)
primitives_sec = extract_primitives_section(report_lines)
primitives = parse_primitives_section(primitives_sec) if primitives_sec else {}
return {"optimized": optimized, "primitives": primitives}
else:
print("⚠️ Report file missing despite success message.")
return None
else:
print("Synthesis did not complete successfully.")
print(f"❌ Synthesis Failed (Logic Error) for {top_module}")
# Print the last 10 lines of the log to help debug
print("\n".join(result.stdout.splitlines()[-10:]))
return None
def run_resource_usage():
# Load the original JSON.
input_json_file = "solutions.json" # Update this file name if needed.
input_json_file = "solutions.json"
if not os.path.exists(input_json_file):
print(f"Error: {input_json_file} not found.")
return
with open(input_json_file, "r") as f:
data = json.load(f)
# Traverse all top-level keys (e.g., "4o") and all subcategories.
for top_key, top_value in data.items():
# print(top_value.keys())
# exit()
# top_value should be a dict with categories (e.g., "Combinational Logic", "Finite State Machines", etc.)
for category, module_list in top_value.items():
# if category == "Combinational Logic":
# continue
print(f"Loaded {input_json_file}. Scanning for passing solutions...")
found_any = False
for top_key, top_value in data.items(): # e.g. "gpt-4"
for category, module_list in top_value.items(): # e.g. "Combinational Logic"
for module in module_list:
for sol in module["solutions"]:
if sol.get("pass", "").strip().lower() == "true":
for idx, sol in enumerate(module["solutions"]):
# DEBUG PRINT: Show us the status
status = sol.get("pass", "MISSING").strip().lower()
if status == "true":
found_any = True
print(f"🚀 MATCH: Synthesizing {module['module']} (Sol #{idx})...")
solution_code = sol["solution"]
print(f"Running synthesis for module '{module['module']}' in category '{category}'")
resource_usage = run_synthesis(solution_code)
if resource_usage:
sol["resource usage"] = resource_usage
print(f" ✅ Result: {resource_usage['optimized']['LUT']} LUTs")
else:
sol["resource usage"] = {"optimized": {}, "primitives": {}}
print(f" ❌ Synthesis Failed internally.")
# Save immediately
with open("solutions.json", "w") as f:
json.dump(data, f, indent=4)
else:
sol["resource usage"] = {"optimized": {}, "primitives": {}}
# Write the updated JSON (with resource usage added) to a new file.
output_json_file = "solutions.json"
with open(output_json_file, "w") as f:
json.dump(data, f, indent=4)
print(f"Updated JSON written to {output_json_file}")
# Tell the user we are skipping
print(f"⏭️ Skipping {module['module']} (Status: '{status}')")
if not found_any:
print("\n⚠️ WARNING: No passing solutions found! Synthesis only runs on code that passed simulation.")
print("To force a test, edit 'solutions.json' and change one 'pass' value to 'true'.")
else:
print("\n✅ All Resource Usage checks completed.")
if __name__ == "__main__":
run_resource_usage()

3
setup.py
View File

@@ -1,7 +1,8 @@
import argparse
import subprocess
from generate_solutions import generate_solutions
from functional_correctness import run_functional_correctness, run_resource_usage
from functional_correctness import run_functional_correctness
from resource_usage import run_resource_usage
def main():
parser = argparse.ArgumentParser(description="Command-line interface for Verilog solution generation and evaluation.")