sBubshait/ResBench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: fixes and configuration to path

Browse Source
This commit is contained in:
Saleh Bubshait
2025-12-17 18:16:44 +00:00
parent a4a9644f9e
commit abfd851c59
4 changed files with 130 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
functional_correctness.py
View File

@@ -10,7 +10,12 @@ TEMP_VERILOG_FILE = "temp.v"
TEMP_TESTBENCH_FILE = "testbench.v" TEMP_TESTBENCH_FILE = "testbench.v"
TCL_SCRIPT_FILE = "run_testbench.tcl" TCL_SCRIPT_FILE = "run_testbench.tcl"
def write_tcl(): # --- CONFIGURATION ---
# Default fallback path if env var is missing
DEFAULT_VIVADO_PATH = "/tools/Xilinx/Vivado/2023.1/bin"
# FIX 1: Add 'top_module' as an argument so the function can use it
def write_tcl(top_module):
# Generate the TCL script for Vivado # Generate the TCL script for Vivado
tcl_commands = f""" tcl_commands = f"""
create_project temp_project ./temp_project -force -part xc7z020clg400-1 create_project temp_project ./temp_project -force -part xc7z020clg400-1
@@ -33,14 +38,15 @@ def extract_top_module_name(testbench_file):
for line in file: for line in file:
match = re.search(r'\s*module\s+(\w+)\s*;', line) match = re.search(r'\s*module\s+(\w+)\s*;', line)
if match: if match:
print(match.group(1))
return match.group(1) # Extract module name return match.group(1) # Extract module name
return None # Return None if no module found return None # Return None if no module found
def run_functional_correctness(): def run_functional_correctness():
# Load JSON files # Load JSON files
if not os.path.exists(SOLUTIONS_FILE):
print(f"Error: {SOLUTIONS_FILE} not found.")
return
with open(SOLUTIONS_FILE, "r", encoding="utf-8") as file: with open(SOLUTIONS_FILE, "r", encoding="utf-8") as file:
solutions_data = json.load(file) solutions_data = json.load(file)
@@ -56,34 +62,31 @@ def run_functional_correctness():
if module_name and testbench_code: if module_name and testbench_code:
module_testbenches[module_name] = testbench_code module_testbenches[module_name] = testbench_code
# print(module_testbenches.keys()) # FIX 2: Handle Linux Path correctly (No .bat)
vivado_path_env = os.environ.get("vivado")
# Get Vivado path from environment variable
vivado_path = os.environ.get("vivado")
if not vivado_path:
raise EnvironmentError("Vivado environment variable not set.")
vivado_path = os.path.join(vivado_path, "vivado.bat")
if vivado_path_env:
vivado_bin = os.path.join(vivado_path_env, "vivado")
elif os.path.exists(os.path.join(DEFAULT_VIVADO_PATH, "vivado")):
vivado_bin = os.path.join(DEFAULT_VIVADO_PATH, "vivado")
else:
vivado_bin = "vivado" # Hope it's in PATH
print(vivado_bin)
# Iterate over solutions and test them # Iterate over solutions and test them
for model, categories in solutions_data.items(): for model, categories in solutions_data.items():
for category, modules in categories.items(): for category, modules in categories.items():
for module_entry in modules: for module_entry in modules:
module_name = module_entry["module"] module_name = module_entry["module"]
# print(module_name)
# print(module_name in module_testbenches.keys())
if module_name not in module_testbenches: if module_name not in module_testbenches:
print(f"Skipping {module_name}: No testbench found.") print(f"Skipping {module_name}: No testbench found.")
continue continue
testbench_code = module_testbenches[module_name] testbench_code = module_testbenches[module_name]
solutions = module_entry["solutions"] solutions = module_entry["solutions"]
# Iterate over all solutions # Iterate over all solutions
for solution_entry in solutions: for idx, solution_entry in enumerate(solutions):
verilog_code = solution_entry["solution"] verilog_code = solution_entry["solution"]
# Write the Verilog design to a file # Write the Verilog design to a file
@@ -101,31 +104,34 @@ def run_functional_correctness():
solution_entry["pass"] = "Error: Could not extract top module." solution_entry["pass"] = "Error: Could not extract top module."
continue continue
print(f"Testing module: {module_name} (Top Module: {top_module})") print(f"Testing {module_name} (Solution {idx+1})...")
write_tcl() # FIX 3: Pass the variable to the function
write_tcl(top_module)
# Run Vivado in batch mode # Run Vivado in batch mode
print(f"Running Vivado simulation for {module_name}...") try:
process = subprocess.run([vivado_path, "-mode", "batch", "-source", TCL_SCRIPT_FILE], capture_output=True, text=True) process = subprocess.run([vivado_bin, "-mode", "batch", "-source", TCL_SCRIPT_FILE], capture_output=True, text=True)
# Capture output logs
output_log = process.stdout + "\n" + process.stderr output_log = process.stdout + "\n" + process.stderr
print(output_log) except FileNotFoundError:
print("CRITICAL ERROR: Vivado executable not found. Check path.")
return
test_passed = "All tests passed" in output_log test_passed = "All tests passed" in output_log
# Determine pass/fail status # Determine pass/fail status
if test_passed: if test_passed:
solution_entry["pass"] = "true" solution_entry["pass"] = "true"
print(f" ✅ PASS")
else: else:
# Extract relevant error messages solution_entry["pass"] = "false" # Keep it simple for now
error_lines = "\n".join(line for line in output_log.split("\n") if "error" or "fail" in line.lower()) print(f" ❌ FAIL")
solution_entry["pass"] = error_lines if error_lines else "Test failed somehow"
print(f"Test result for {module_name}: {'PASS' if test_passed else 'FAIL'}") # Save results incrementally
# Save results after testing each module
with open(SOLUTIONS_FILE, "w", encoding="utf-8") as file: with open(SOLUTIONS_FILE, "w", encoding="utf-8") as file:
json.dump(solutions_data, file, indent=4) json.dump(solutions_data, file, indent=4)
print("All tests completed.") print("All tests completed.")
if __name__ == "__main__":
run_functional_correctness()

138
resource_usage.py
View File

@@ -3,6 +3,10 @@ import subprocess
import os import os
import re import re
# --- CONFIGURATION ---
# If your environment variable isn't set, this default path will be used.
DEFAULT_VIVADO_PATH = "/tools/Xilinx/Vivado/2023.1/bin"
def extract_module_name(verilog_code): def extract_module_name(verilog_code):
""" """
Extract the module name from the Verilog code. Extract the module name from the Verilog code.
@@ -62,10 +66,6 @@ def extract_primitives_section(lines):
def parse_primitives_section(lines): def parse_primitives_section(lines):
""" """
Parses the primitives section lines to accumulate resource usage. Parses the primitives section lines to accumulate resource usage.
Returns a dictionary with keys: LUT, FF, DSP, BRAM, IO.
In this example:
- For LUT: sums up any primitive whose name starts with "LUT" (e.g., LUT2, LUT3, ...)
- For IO: sums the usage of IBUF and OBUF.
""" """
resources = {"LUT": 0, "FF": 0, "DSP": 0, "BRAM": 0, "IO": 0} resources = {"LUT": 0, "FF": 0, "DSP": 0, "BRAM": 0, "IO": 0}
for line in lines: for line in lines:
@@ -85,110 +85,140 @@ def parse_primitives_section(lines):
resources["LUT"] += used resources["LUT"] += used
if ref_name in ("IBUF", "OBUF"): if ref_name in ("IBUF", "OBUF"):
resources["IO"] += used resources["IO"] += used
# (Add additional processing for FF, DSP, BRAM if necessary.)
return resources return resources
def run_synthesis(solution_code): def run_synthesis(solution_code):
""" """
Writes the given Verilog solution to a temporary file, Runs Vivado synthesis on the provided code.
creates a Tcl script for Vivado to run synthesis and generate a utilization report,
runs Vivado in batch mode, and parses the resource usage report.
Returns a dictionary with keys "optimized" and "primitives" containing resource usage.
""" """
# Write the Verilog code to a temporary file. # 1. Write the Verilog code to a temporary file.
verilog_file = "temp.v" verilog_file = "temp.v"
with open(verilog_file, "w") as f: with open(verilog_file, "w") as f:
f.write(solution_code) f.write(solution_code)
# Extract the module name from the solution code. # 2. Extract the module name
top_module = extract_module_name(solution_code) top_module = extract_module_name(solution_code)
print(top_module)
if top_module is None: if top_module is None:
print("Could not extract module name; using 'temp_top' as a default.") print("Could not extract module name; using 'temp_top' as a default.")
top_module = "temp_top" top_module = "temp_top"
vivado_project = "temp_project" # 3. Resolve Vivado Path (Linux Fix)
tcl_script = "synthesis_script.tcl"
# Get the Vivado installation path from the environment variable.
vivado_path_env = os.environ.get("vivado") vivado_path_env = os.environ.get("vivado")
if vivado_path_env is None:
print("Error: 'vivado' environment variable is not set.")
return None
vivado_path = os.path.join(vivado_path_env, "vivado.bat")
# Create the Vivado Tcl script. # If env var is missing, try the default path, or search system path
if vivado_path_env:
vivado_bin = os.path.join(vivado_path_env, "vivado") # NO .bat
elif os.path.exists(os.path.join(DEFAULT_VIVADO_PATH, "vivado")):
vivado_bin = os.path.join(DEFAULT_VIVADO_PATH, "vivado")
else:
# Last resort: assume it's in the system PATH
vivado_bin = "vivado"
tcl_script = "synthesis_script.tcl"
vivado_project = "temp_project"
# 4. Create the Vivado Tcl script.
tcl_commands = f""" tcl_commands = f"""
create_project {vivado_project} -force -part xc7z020clg400-1 create_project {vivado_project} -force -part xc7z020clg400-1
add_files {verilog_file} add_files {verilog_file}
set_property top {top_module} [current_fileset] set_property top {top_module} [current_fileset]
# Run synthesis only (no simulation) # Run synthesis only (no simulation)
synth_design -top {top_module} synth_design -top {top_module} -part xc7z020clg400-1
# Generate resource utilization report # Generate resource utilization report
report_utilization -file resource_usage.rpt report_utilization -file resource_usage.rpt
quit exit
""" """
with open(tcl_script, "w") as file: with open(tcl_script, "w") as file:
file.write(tcl_commands) file.write(tcl_commands)
# Run Vivado in batch mode using the generated Tcl script. # 5. Run Vivado
print(f"Starting Synthesis for {top_module} using {vivado_bin}...")
try: try:
# Added stdout=PIPE so we can see the output if it crashes
result = subprocess.run( result = subprocess.run(
[vivado_path, "-mode", "batch", "-source", tcl_script], [vivado_bin, "-mode", "batch", "-source", tcl_script],
capture_output=True, text=True, check=True capture_output=True, text=True, check=True
) )
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print("Synthesis failed:", e) print(f"❌ Synthesis CRASHED for {top_module}")
print("--- Error Log ---")
print(e.stderr) # Print the actual error from Vivado
print("-----------------")
return None return None
print(result.stdout) except FileNotFoundError:
# Check for the success message in the output. print(f"❌ CRITICAL ERROR: Could not find Vivado executable at: {vivado_bin}")
return None
# 6. Check results
if "Finished Writing Synthesis Report" in result.stdout: if "Finished Writing Synthesis Report" in result.stdout:
# Read the resource utilization report. print(f"✅ Synthesis Success: {top_module}")
if os.path.exists("resource_usage.rpt"):
with open("resource_usage.rpt", "r") as f: with open("resource_usage.rpt", "r") as f:
report_lines = f.readlines() report_lines = f.readlines()
optimized_resources = parse_optimized(report_lines) optimized = parse_optimized(report_lines)
primitives_section = extract_primitives_section(report_lines) primitives_sec = extract_primitives_section(report_lines)
primitives_resources = (parse_primitives_section(primitives_section) primitives = parse_primitives_section(primitives_sec) if primitives_sec else {}
if primitives_section else {}) return {"optimized": optimized, "primitives": primitives}
return {"optimized": optimized_resources, "primitives": primitives_resources}
else: else:
print("Synthesis did not complete successfully.") print("⚠️ Report file missing despite success message.")
return None
else:
print(f"❌ Synthesis Failed (Logic Error) for {top_module}")
# Print the last 10 lines of the log to help debug
print("\n".join(result.stdout.splitlines()[-10:]))
return None return None
def run_resource_usage(): def run_resource_usage():
# Load the original JSON. input_json_file = "solutions.json"
input_json_file = "solutions.json" # Update this file name if needed.
if not os.path.exists(input_json_file):
print(f"Error: {input_json_file} not found.")
return
with open(input_json_file, "r") as f: with open(input_json_file, "r") as f:
data = json.load(f) data = json.load(f)
# Traverse all top-level keys (e.g., "4o") and all subcategories. print(f"Loaded {input_json_file}. Scanning for passing solutions...")
for top_key, top_value in data.items(): found_any = False
# print(top_value.keys())
# exit() for top_key, top_value in data.items(): # e.g. "gpt-4"
# top_value should be a dict with categories (e.g., "Combinational Logic", "Finite State Machines", etc.) for category, module_list in top_value.items(): # e.g. "Combinational Logic"
for category, module_list in top_value.items():
# if category == "Combinational Logic":
# continue
for module in module_list: for module in module_list:
for sol in module["solutions"]: for idx, sol in enumerate(module["solutions"]):
if sol.get("pass", "").strip().lower() == "true":
# DEBUG PRINT: Show us the status
status = sol.get("pass", "MISSING").strip().lower()
if status == "true":
found_any = True
print(f"🚀 MATCH: Synthesizing {module['module']} (Sol #{idx})...")
solution_code = sol["solution"] solution_code = sol["solution"]
print(f"Running synthesis for module '{module['module']}' in category '{category}'")
resource_usage = run_synthesis(solution_code) resource_usage = run_synthesis(solution_code)
if resource_usage: if resource_usage:
sol["resource usage"] = resource_usage sol["resource usage"] = resource_usage
print(f" ✅ Result: {resource_usage['optimized']['LUT']} LUTs")
else: else:
sol["resource usage"] = {"optimized": {}, "primitives": {}} sol["resource usage"] = {"optimized": {}, "primitives": {}}
else: print(f" ❌ Synthesis Failed internally.")
sol["resource usage"] = {"optimized": {}, "primitives": {}}
# Write the updated JSON (with resource usage added) to a new file. # Save immediately
output_json_file = "solutions.json" with open("solutions.json", "w") as f:
with open(output_json_file, "w") as f:
json.dump(data, f, indent=4) json.dump(data, f, indent=4)
print(f"Updated JSON written to {output_json_file}")
else:
# Tell the user we are skipping
print(f"⏭️ Skipping {module['module']} (Status: '{status}')")
if not found_any:
print("\n⚠️ WARNING: No passing solutions found! Synthesis only runs on code that passed simulation.")
print("To force a test, edit 'solutions.json' and change one 'pass' value to 'true'.")
else:
print("\n✅ All Resource Usage checks completed.")
if __name__ == "__main__":
run_resource_usage()

3
setup.py
View File

@@ -1,7 +1,8 @@
import argparse import argparse
import subprocess import subprocess
from generate_solutions import generate_solutions from generate_solutions import generate_solutions
from functional_correctness import run_functional_correctness, run_resource_usage from functional_correctness import run_functional_correctness
from resource_usage import run_resource_usage
def main(): def main():
parser = argparse.ArgumentParser(description="Command-line interface for Verilog solution generation and evaluation.") parser = argparse.ArgumentParser(description="Command-line interface for Verilog solution generation and evaluation.")