diff --git a/benchmarks/dhrystone_run_multiple.py b/benchmarks/dhrystone_run_multiple.py new file mode 100644 index 0000000..c56bffc --- /dev/null +++ b/benchmarks/dhrystone_run_multiple.py @@ -0,0 +1,108 @@ +import argparse +import os +import shutil +import subprocess +import time +from pathlib import Path + + +def run_command(command, cwd=None): + """Run a shell command in the specified directory and return its output.""" + result = subprocess.run( + command, + shell=True, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + return result.stdout.decode("utf-8"), result.stderr.decode("utf-8") + + +def build_test_cases(makefile_dir, iterations): + """Run the Makefile with the specified iterations.""" + make_command = f"make clean && make ITERATIONS={iterations}" + stdout, stderr = run_command(make_command, cwd=makefile_dir) + if stderr: + raise RuntimeError(f"Error during make: {stderr}") + + +def main(simulator_path, makefile_dir): + # Directory for generated test cases + generated_dir = makefile_dir.parent / "workspace" + os.makedirs(generated_dir, exist_ok=True) + + # Define the iterations + iterations_list = [10, 20, 30] + # 15 value up to 6.000.000 evenly apart on a log scale + iterations_list = [ + 1, + 2, + 4, + 8, + 17, + 34, + 69, + 141, + 287, + 582, + 1182, + 2401, + 4878, + 9910, + 20133, + 40914, + 83103, + 168830, + 343042, + 696712, + 1414641, + 2874878, + 5837995, + ] + + for iteration in iterations_list: + try: + # Update the Makefile with the current ITERATIONS value + build_test_cases(makefile_dir, iteration) + except RuntimeError as e: + print(f"Error during compilation with ITERATIONS={iteration}: {e}") + continue + + # Run the simulator with the generated test case + exe = makefile_dir / "dhrystone.elf" + if not exe.is_file(): + exit(f"{exe} does not exist") + verbose_exe = generated_dir / "bin" / f"dhrystone_{iteration}.elf" + os.makedirs(verbose_exe.parent, exist_ok=True) + shutil.copy(exe, verbose_exe) + backends = ["interp", "llvm", "tcc", "asmjit"] + for backend in backends: + log_file = os.path.join(generated_dir, f"{backend}_{iteration}.log") + + sim_command = f"{simulator_path} -f {exe} --backend {backend}" + start_time = time.time() + sim_stdout, sim_stderr = run_command(sim_command) + end_time = time.time() + elapsed_time = end_time - start_time + + # Save the output to the logfile + with open(log_file, "w", encoding="utf8") as f: + f.write(sim_stdout) + if sim_stderr: + f.write(f"\nErrors:\n{sim_stderr}") + + print( + f"Ran {backend} in {elapsed_time:.2f} s, Output saved to {backend}_{iteration}.log" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run simulations with generated test cases." + ) + parser.add_argument("simulator_path", help="Path to the simulator executable.") + + args = parser.parse_args() + dhrystone_path = Path(__file__).parent / "dhrystone" + main(args.simulator_path, dhrystone_path) diff --git a/benchmarks/visualize_runtimes.py b/benchmarks/visualize_runtimes.py new file mode 100644 index 0000000..f02588a --- /dev/null +++ b/benchmarks/visualize_runtimes.py @@ -0,0 +1,86 @@ +import argparse +import os +import re +from pathlib import Path + +import plotly.express as px +import yaml + + +def parse_logs(log_dir): + results = [] + for filename in os.listdir(log_dir): + if filename.endswith(".log"): + filepath = os.path.join(log_dir, filename) + with open(filepath, "r", encoding="utf8") as file: + for line in file: + if ( + "Executed" in line + and "instructions" in line + and "during" in line + and "resulting in" in line + ): + parts = line.split() + instructions = int(parts[3]) + time_idx = parts.index("during") + 1 + time = int(parts[time_idx].rstrip("ms")) + mips_idx = parts.index("resulting") + 2 + mips = float(parts[mips_idx].rstrip("MIPS")) + backend, iterations, _ = re.split(r"[_.]", filename) + + results.append( + { + "backend": backend, + "run_count": int(iterations), + "instructions": instructions, + "time": time, + "mips": mips, + } + ) + return results + + +def write_yaml(results, output_file): + with open(output_file, "w", encoding="utf8") as file: + yaml.dump(results, file) + + +def visualize_mips_over_instructions(yaml_file): + # Read data from YAML file + with open(yaml_file, "r", encoding="utf8") as file: + data = yaml.safe_load(file) + + # Extract instructions and MIPS values + run_count = [entry["run_count"] for entry in data] + mips = [entry["mips"] for entry in data] + backends = [entry["backend"] for entry in data] + # Create scatter plot using Plotly Express + fig = px.line( + x=run_count, + y=mips, + color=backends, + labels={"x": "Dhrystone Iterations", "y": "MIPS", "color": "Backend"}, + title="MIPS over Amount of Dhrystone Iterations", + log_x=True, + ) + fig.show() + + +def main(): + parser = argparse.ArgumentParser( + description=""" + Parse log files and extract relevant information. Create a 'results.yaml' file and visualize it. + Intended to be run after 'dhrystone_run_multiple.py'""" + ) + parser.add_argument("log_dir", help="Path to the directory containing log files.") + args = parser.parse_args() + + result_file = Path(__file__).parent / "results.yaml" + if not result_file.is_file(): + results = parse_logs(args.log_dir) + write_yaml(results, result_file) + visualize_mips_over_instructions(result_file) + + +if __name__ == "__main__": + main()