#!/usr/bin/env python3 import os import pickle import re import shutil import socket import sys from copy import deepcopy from glob import glob from itertools import chain from pathlib import Path from subprocess import STDOUT, CalledProcessError, check_call, check_output import requests from kerncraft.incore_model import ( asm_instrumentation, iaca_analyse_instrumented_binary, llvm_mca_analyse_instrumented_assembly, osaca_analyse_instrumented_assembly, parse_asm, ) from kerncraft.models import benchmark from osaca.osaca import reduce_to_section # Scaling of inner dimension for 1D, 2D and 3D kernels # * consider kernels to be compiled with multiple compilers and different options # * find best performing run (min cy/it over all runs) # * statistics on performance overall (cy/it over inner length) # * validate that L2 traffic is neglegible # * measure other performance metrics, such as port utilization (optionally) # * scale to highlevel iterations # Collect inner loop body assembly for each kernel/compiler/options combination # * analyze with OSACA, IACA and LLVM-MCA hosts_arch_map = { r"skylakesp2": "SKX", r"ivyep1": "IVB", r"naples1": "ZEN", r"rome1": "ZEN2", r"warmup": "TX2", r"qp4-node-[0-9]+": "A64FX", r"milan1": "ZEN3", r"f0[0-9]+.nhr.fau.de": "ICX", r"f2[0-9]+.nhr.fau.de": "SPR", r"gracesup1": "V2", r"genoa[1-3]": "ZEN4", } arch_info = { "SPR": { "OSACA": "SPR", "LLVM-MCA": "-mcpu=sapphirerapids", "Ithemal": None, "IACA": None, "isa": "x86", "perfevents": [], "cflags": { "icx": { "Ofast": ( "-Ofast -fno-alias -march=sapphirerapids -qopt-zmm-usage=high -nolib-inline " "-ffreestanding" ).split(), "O3": ( "-O3 -fno-alias -march=sapphirerapids -qopt-zmm-usage=high -nolib-inline " "-ffreestanding" ).split(), "O2": ( "-O2 -fno-alias -march=sapphirerapids -qopt-zmm-usage=high -nolib-inline " "-ffreestanding" ).split(), "O1": ( "-O1 -fno-alias -march=sapphirerapids -qopt-zmm-usage=high -nolib-inline " "-ffreestanding" ).split(), }, "clang": { "Ofast": "-Ofast -march=sapphirerapids -ffreestanding".split(), "O3": "-O3 -march=sapphirerapids -ffreestanding".split(), "O2": "-O2 -march=sapphirerapids -ffreestanding".split(), "O1": "-O1 -march=sapphirerapids -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=sapphirerapids -lm -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=sapphirerapids -lm -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=sapphirerapids -lm -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=sapphirerapids -lm -ffreestanding -falign-loops=16".split(), }, }, }, "ICX": { "IACA": "SKX", "OSACA": "ICX", "LLVM-MCA": "-mcpu=icelake-server", "Ithemal": None, "isa": "x86", "perfevents": [], "cflags": { "icc": { "Ofast": ( "-Ofast -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), "O3": ( "-O3 -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), "O2": ( "-O2 -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), "O1": ( "-O1 -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), }, "clang": { "Ofast": "-Ofast -march=icelake-server -ffreestanding".split(), "O3": "-O3 -march=icelake-server -ffreestanding".split(), "O2": "-O2 -march=icelake-server -ffreestanding".split(), "O1": "-O1 -march=icelake-server -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=icelake-server -lm -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=icelake-server -lm -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=icelake-server -lm -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=icelake-server -lm -ffreestanding -falign-loops=16".split(), }, }, }, "SKX": { "prepare": ["likwid-setFrequencies -f 2.4 -t 0".split()], "IACA": "SKX", "OSACA": "SKX", "LLVM-MCA": "-mcpu=skylake-avx512", "Ithemal": "skl", "isa": "x86", "perfevents": [], "cflags": { "icc": { "Ofast": ( "-Ofast -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), "O3": ( "-O3 -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), "O2": ( "-O2 -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), "O1": ( "-O1 -fno-alias -xCORE-AVX512 -qopt-zmm-usage=high -nolib-inline " "-ffreestanding -falign-loops" ).split(), }, "clang": { "Ofast": "-Ofast -march=skylake-avx512 -ffreestanding".split(), "O3": "-O3 -march=skylake-avx512 -ffreestanding".split(), "O2": "-O2 -march=skylake-avx512 -ffreestanding".split(), "O1": "-O1 -march=skylake-avx512 -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=skylake-avx512 -lm -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=skylake-avx512 -lm -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=skylake-avx512 -lm -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=skylake-avx512 -lm -ffreestanding -falign-loops=16".split(), }, }, }, "IVB": { "prepare": ["likwid-setFrequencies -f 3.0 -t 0".split()], "IACA": "IVB", "OSACA": "IVB", "LLVM-MCA": "-mcpu=ivybridge", "Ithemal": "ivb", "isa": "x86", "perfevents": [], "cflags": { "icc": { "Ofast": ( "-Ofast -xAVX -fno-alias -nolib-inline -ffreestanding -falign-loops" ).split(), "O3": "-O3 -xAVX -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O2": "-O2 -xAVX -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O1": "-O1 -xAVX -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), }, "clang": { "Ofast": "-Ofast -mavx -ffreestanding".split(), "O3": "-O3 -mavx -ffreestanding".split(), "O2": "-O2 -mavx -ffreestanding".split(), "O1": "-O1 -mavx -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=corei7-avx -lm -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=corei7-avx -lm -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=corei7-avx -lm -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=corei7-avx -lm -ffreestanding -falign-loops=16".split(), }, }, }, "ZEN": { "prepare": ["likwid-setFrequencies -f 2.3 -t 0".split()], "IACA": None, "OSACA": "ZEN1", "LLVM-MCA": "-mcpu=znver1", "Ithemal": None, "isa": "x86", "perfevents": [], "cflags": { "clang": { "Ofast": "-Ofast -march=znver1 -ffreestanding".split(), "O3": "-O3 -march=znver1 -ffreestanding".split(), "O2": "-O2 -march=znver1 -ffreestanding".split(), "O1": "-O1 -march=znver1 -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=znver1 -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=znver1 -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=znver1 -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=znver1 -ffreestanding -falign-loops=16".split(), }, "icc": { "Ofast": ( "-Ofast -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops" ).split(), "O3": "-O3 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O2": "-O2 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O1": "-O1 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), }, }, }, "ZEN2": { "prepare": ["likwid-setFrequencies -f 2.35 -t 0".split()], "IACA": None, "OSACA": "ZEN2", "LLVM-MCA": "-mcpu=znver2", "Ithemal": None, "isa": "x86", "perfevents": [], "cflags": { "clang": { "Ofast": "-Ofast -march=znver2 -ffreestanding".split(), "O3": "-O3 -march=znver2 -ffreestanding".split(), "O2": "-O2 -march=znver2 -ffreestanding".split(), "O1": "-O1 -march=znver2 -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=znver2 -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=znver2 -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=znver2 -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=znver2 -ffreestanding -falign-loops=16".split(), }, "icc": { "Ofast": ( "-Ofast -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops" ).split(), "O3": "-O3 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O2": "-O2 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O1": "-O1 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), }, }, }, "ZEN3": { "prepare": ["likwid-setFrequencies -f 2.1 -t 0".split()], "IACA": None, "OSACA": "ZEN3", "LLVM-MCA": "-mcpu=znver3", "Ithemal": None, "isa": "x86", "perfevents": [], "L2_volume_metric": "L2 bandwidth [MBytes/s]", "cflags": { "clang-15": { "Ofast": "-Ofast -march=znver3 -ffreestanding".split(), "O3": "-O3 -march=znver3 -ffreestanding".split(), "O2": "-O2 -march=znver3 -ffreestanding".split(), "O1": "-O1 -march=znver3 -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=znver3 -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=znver3 -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=znver3 -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=znver3 -ffreestanding -falign-loops=16".split(), }, "icc": { "Ofast": ( "-Ofast -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops" ).split(), "O3": "-O3 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O2": "-O2 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O1": "-O1 -xAVX2 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), }, }, }, "ZEN4": { "IACA": None, "OSACA": "ZEN4", "LLVM-MCA": "-mcpu=znver4", "Ithemal": None, "isa": "x86", "perfevents": [], "L2_volume_metric": "L2 bandwidth [MBytes/s]", "cflags": { "clang": { "Ofast": "-Ofast -march=znver4 -ffreestanding".split(), "O3": "-O3 -march=znver4 -ffreestanding".split(), "O2": "-O2 -march=znver4 -ffreestanding".split(), "O1": "-O1 -march=znver4 -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=znver4 -ffreestanding -falign-loops=16".split(), "O3": "-O3 -march=znver4 -ffreestanding -falign-loops=16".split(), "O2": "-O2 -march=znver4 -ffreestanding -falign-loops=16".split(), "O1": "-O1 -march=znver4 -ffreestanding -falign-loops=16".split(), }, "icx": { "Ofast": ( "-Ofast -xCORE-AVX512 -fno-alias -nolib-inline -ffreestanding -falign-loops" ).split(), "O3": "-O3 -xCORE-AVX512 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O2": "-O2 -xCORE-AVX512 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), "O1": "-O1 -xCORE-AVX512 -fno-alias -nolib-inline -ffreestanding -falign-loops".split(), }, }, }, "TX2": { "Clock [MHz]": 2200, # reading out via perf. counters is not supported "IACA": None, "OSACA": "TX2", "assign_optimal_throughput": True, "LLVM-MCA": "-mcpu=thunderx2t99 -march=aarch64", "Ithemal": None, "isa": "aarch64", "perfevents": [], "cflags": { "clang": { "Ofast": "-Ofast -target aarch64-unknown-linux-gnu -ffreestanding".split(), "O3": "-O3 -target aarch64-unknown-linux-gnu -ffreestanding".split(), "O2": "-O2 -target aarch64-unknown-linux-gnu -ffreestanding".split(), "O1": "-O1 -target aarch64-unknown-linux-gnu -ffreestanding".split(), }, "gcc": { "Ofast": "-Ofast -march=armv8.1-a -ffreestanding".split(), "O3": "-O3 -march=armv8.1-a -ffreestanding".split(), "O2": "-O2 -march=armv8.1-a -ffreestanding".split(), "O1": "-O1 -march=armv8.1-a -ffreestanding".split(), }, }, }, "A64FX": { "Clock [MHz]": 1800, # reading out via perf. counters is not supported "L2_volume_metric": "L1<->L2 data volume [GBytes]", "IACA": None, "OSACA": "A64FX", "assign_optimal_throughput": False, "LLVM-MCA": "-mcpu=a64fx -march=aarch64", "Ithemal": None, "isa": "aarch64", "perfevents": [], "cflags": { "gcc": { "Ofast": "-Ofast -msve-vector-bits=512 -march=armv8.2-a+sve -ffreestanding".split(), "O3": "-O3 -msve-vector-bits=512 -march=armv8.2-a+sve -ffreestanding".split(), "O2": "-O2 -msve-vector-bits=512 -march=armv8.2-a+sve -ffreestanding".split(), "O1": "-O1 -msve-vector-bits=512 -march=armv8.2-a+sve -ffreestanding".split(), }, "clang": { "Ofast": "-Ofast -target aarch64-unknown-linux-gnu -ffreestanding".split(), "O3": "-O3 -target aarch64-unknown-linux-gnu -ffreestanding".split(), "O2": "-O2 -target aarch64-unknown-linux-gnu -ffreestanding".split(), "O1": "-O1 -target aarch64-unknown-linux-gnu -ffreestanding".split(), }, }, }, "V2": { "Clock [MHz]": 3400, # reading out via perf. counters is not supported "L2_volume_metric": "L1<->L2 data volume [GBytes]", "IACA": None, "OSACA": "V2", "assign_optimal_throughput": True, "LLVM-MCA": "-mcpu=neoverse-v2 -march=aarch64", "Ithemal": None, "isa": "aarch64", "perfevents": [], "cflags": { "gcc": { "Ofast": "-Ofast -msve-vector-bits=128 -march=armv9-a+sve2 -ffreestanding".split(), "O3": "-O3 -msve-vector-bits=128 -march=armv9-a+sve2 -ffreestanding".split(), "O2": "-O2 -msve-vector-bits=128 -march=armv9-a+sve2 -ffreestanding".split(), "O1": "-O1 -msve-vector-bits=128 -march=armv9-a+sve2 -ffreestanding".split(), }, "armclang": { "Ofast": "-Ofast -target aarch64-unknown-linux-gnu -march=armv9-a+sve2 -mcpu=neoverse-v2 -ffreestanding".split(), "O3": "-O3 -target aarch64-unknown-linux-gnu -march=armv9-a+sve2 -mcpu=neoverse-v2 -ffreestanding".split(), "O2": "-O2 -target aarch64-unknown-linux-gnu -march=armv9-a+sve2 -mcpu=neoverse-v2 -ffreestanding".split(), "O1": "-O1 -target aarch64-unknown-linux-gnu -march=armv9-a+sve2 -mcpu=neoverse-v2 -ffreestanding".split(), }, }, }, } def get_current_arch(): hostname = socket.gethostname() if hostname in hosts_arch_map: return hosts_arch_map[hostname] for matchstr, arch in hosts_arch_map.items(): if re.match(matchstr, hostname): return arch # raise KeyError(f"{hostname} not matched in hosts_arch_map.") return None def get_kernels(kernels=None): if kernels is None: kernels = [] for f in glob("kernels/*.c"): f = f.rsplit(".", 1)[0].split("/", 1)[1] if f == "dummy": continue kernels.append(f) return kernels # Columns: # arch # kernel # compiler # cflags_name # element_size # pointer_increment # IACA_raw # IACA_scaled [dict with cy/it] # IACA_scaled_max [float with cy/it] # OSACA_raw # OSACA_scaled [dict with cy/it] # OSACA_scaled_max [float with cy/it] # LLVM-MCA_raw # LLVM-MCA_scaled [dict with cy/it] # LLVM-MCA_scaled_max [float with cy/it] # best_length # best_runtime [cy/it] # L2_traffic [B/it] # allruns [list (length, repetitions, cy/it, L2 B/it)] # perfevents [dict event: counter/it] def build_mark_run_all_kernels(measurements=True, osaca=True, iaca=True, llvm_mca=True): arch = get_current_arch() if arch is None: arches = arch_info.keys() islocal = False else: islocal = True arches = [arch] ainfo = arch_info.get(arch) if "prepare" in ainfo: for cmd in ainfo["prepare"]: check_call(cmd) for arch in arches: ainfo = arch_info.get(arch) print(arch) data_path = Path(f"build/{arch}/data.pkl") if data_path.exists(): with data_path.open("rb") as f: data = pickle.load(f) else: data = [] data_lastsaved = deepcopy(data) for compiler, compiler_cflags in ainfo["cflags"].items(): if not shutil.which(compiler) and islocal: print(compiler, "not found in path! Skipping...") continue for cflags_name, cflags in compiler_cflags.items(): for kernel in get_kernels(): print( f"{kernel:<15} {arch:>5} {compiler:>5} {cflags_name:>6}", end=": ", flush=True, ) row = list( [ r for r in data if r["arch"] == arch and r["kernel"] == kernel and r["compiler"] == compiler and r["cflags_name"] == cflags_name ] ) if row: row = row[0] else: row = { "arch": arch, "kernel": kernel, "compiler": compiler, "cflags_name": cflags_name, "element_size": 8, } data.append(row) # Build print("build", end="", flush=True) asm_path, exec_path, overwrite = build_kernel( kernel, arch, compiler, cflags, cflags_name, dontbuild=not islocal, ) if overwrite: # clear all measurment information row["best_length"] = None row["best_runtime"] = None row["L2_traffic"] = None row["allruns"] = None row["perfevents"] = None # Mark for IACA, OSACA and LLVM-MCA print("mark", end="", flush=True) try: ( marked_asmfile, marked_objfile, row["pointer_increment"], overwrite, ) = mark( asm_path, compiler, cflags, isa=ainfo["isa"], overwrite=overwrite, ) row["marking_error"] = None except ValueError as e: row["marking_error"] = str(e) print(":", e) continue # Always redo the predictions # if overwrite: if True: # clear all model generated information for model in ["IACA", "OSACA", "LLVM-MCA", "Ithemal"]: for k in [ "ports", "prediction", "throughput", "cp", "lcd", "raw", ]: row[model + "_" + k] = None for model in ["IACA", "OSACA", "LLVM-MCA", "Ithemal"]: for k in [ "ports", "prediction", "throughput", "cp", "lcd", "raw", ]: if model + "_" + k not in row: row[model + "_" + k] = None # Analyze with IACA, if requested and configured if iaca and ainfo["IACA"] is not None: print("IACA", end="", flush=True) if not row.get("IACA_ports"): row["IACA_raw"] = iaca_analyse_instrumented_binary( marked_objfile, micro_architecture=ainfo["IACA"] ) row["IACA_ports"] = { k: v / (row["pointer_increment"] / row["element_size"]) for k, v in row["IACA_raw"]["port cycles"].items() } row["IACA_prediction"] = row["IACA_raw"]["throughput"] / ( row["pointer_increment"] / row["element_size"] ) row["IACA_throughput"] = max(row["IACA_ports"].values()) print(". ", end="", flush=True) else: print("! ", end="", flush=True) # Analyze with OSACA, if requested if osaca: print("OSACA", end="", flush=True) if not row.get("OSACA_ports"): row["OSACA_raw"] = osaca_analyse_instrumented_assembly( marked_asmfile, micro_architecture=ainfo["OSACA"], assign_optimal_throughput=ainfo.get( "assign_optimal_throughput", True ), ) row["OSACA_ports"] = { k: v / (row["pointer_increment"] / row["element_size"]) for k, v in row["OSACA_raw"]["port cycles"].items() } row["OSACA_prediction"] = row["OSACA_raw"]["throughput"] / ( row["pointer_increment"] / row["element_size"] ) row["OSACA_throughput"] = max(row["OSACA_ports"].values()) row["OSACA_cp"] = row["OSACA_raw"]["cp_latency"] / ( row["pointer_increment"] / row["element_size"] ) row["OSACA_lcd"] = row["OSACA_raw"]["lcd"] / ( row["pointer_increment"] / row["element_size"] ) print(". ", end="", flush=True) else: print("! ", end="", flush=True) # Analyze with LLVM-MCA, if requested and configured if llvm_mca and ainfo["LLVM-MCA"] is not None: print("LLVM-MCA", end="", flush=True) if not row.get("LLVM-MCA_ports"): row["LLVM-MCA_raw"] = llvm_mca_analyse_instrumented_assembly( marked_asmfile, micro_architecture=ainfo["LLVM-MCA"], isa=ainfo["isa"], ) row["LLVM-MCA_ports"] = { k: v / (row["pointer_increment"] / row["element_size"]) for k, v in row["LLVM-MCA_raw"]["port cycles"].items() } row["LLVM-MCA_prediction"] = row["LLVM-MCA_raw"]["throughput"] / ( row["pointer_increment"] / row["element_size"] ) row["LLVM-MCA_throughput"] = max(row["LLVM-MCA_ports"].values()) row["LLVM-MCA_cp"] = row["LLVM-MCA_raw"]["cp_latency"] / ( row["pointer_increment"] / row["element_size"] ) row["LLVM-MCA_lcd"] = row["LLVM-MCA_raw"]["lcd"] / ( row["pointer_increment"] / row["element_size"] ) print(". ", end="", flush=True) else: print("! ", end="", flush=True) # Analyze with Ithemal, if not running local and configured if ainfo["Ithemal"] is not None and not islocal: print("Ithemal", end="", flush=True) if not row.get("Ithemal_prediction"): with open(marked_asmfile) as f: parsed_code = parse_asm(f.read(), ainfo["isa"]) kernel = reduce_to_section(parsed_code, ainfo["isa"]) row["Ithemal_prediction"] = get_ithemal_prediction( get_intel_style_code(marked_objfile), model=ainfo["Ithemal"], ) print(". ", end="", flush=True) else: print("! ", end="", flush=True) if measurements and islocal: # run measurements if on same hardware print("scale", end="", flush=True) if not row.get("allruns"): # find best length with concurrent L2 measurement scaling_runs, best = scalingrun(exec_path) row["best_length"] = best[0] row["best_runtime"] = best[2] row["L2_traffic"] = best[3] row["allruns"] = scaling_runs print(f"({best[0]}). ", end="", flush=True) else: print( f"({row.get('best_length', None)})! ", end="", flush=True, ) print() # dump to file if data != data_lastsaved: print("saving... ", end="", flush=True) with data_path.open("wb") as f: try: pickle.dump(data, f) data_lastsaved = deepcopy(data) print("saved!") except KeyboardInterrupt: f.seek(0) pickle.dump(data, f) print("saved!") sys.exit() def scalingrun(kernel_exec, total_iterations=25000000, lengths=range(8, 1 * 1024 + 1)): # print('{:>8} {:>10} {:>10}'.format("x", "cy/it", "L2 B/it")) parameters = chain(*[[total_iterations // i, i] for i in lengths]) # TODO use arch specific events and grooup r, o = perfctr(chain([kernel_exec], map(str, parameters)), 1, group="L2") global_infos = {} for m in [re.match(r"(:?([a-z_\-0-9]+):)?([a-z]+): ([a-z\_\-0-9]+)", line) for line in o]: if m is not None: try: v = int(m.group(4)) except ValueError: v = m.group(4) if m.group(1) is None: global_infos[m.group(3)] = v else: r[m.group(2)][m.group(3)] = v results = [] best = (float("inf"), None) for markername, mmetrics in r.items(): kernelname, repetitions, *_, xlength = markername.split("_") repetitions = int(repetitions) xlength = int(xlength) total_iterations = mmetrics["repetitions"] * mmetrics["iterations"] if "Clock [MHz]" in mmetrics: clock_hz = mmetrics["Clock [MHz]"] * 1e6 else: clock_hz = arch_info[get_current_arch()]["Clock [MHz]"] * 1e6 cyperit = mmetrics["Runtime (RDTSC) [s]"] * clock_hz / total_iterations # TODO use arch specific events and grooup if "L2D load data volume [GBytes]" in mmetrics: l2perit = ( ( mmetrics["L2D load data volume [GBytes]"] + mmetrics.get("L2D evict data volume [GBytes]", 0) ) * 1e9 / total_iterations ) else: l2perit = ( mmetrics[arch_info[get_current_arch()]["L2_volume_metric"]] * 1e9 / total_iterations ) results.append((xlength, repetitions, cyperit, l2perit)) if cyperit < best[0]: best = cyperit, results[-1] return results, best[1] def mark(asm_path, compiler, cflags, isa, overwrite=False): # Mark assembly for IACA, OSACA and LLVM-MCA marked_asm_path = Path(asm_path).with_suffix(".marked.s") if not marked_asm_path.exists() or overwrite: overwrite = True with open(asm_path) as fa, open(marked_asm_path, "w") as fm: try: _, pointer_increment = asm_instrumentation(fa, fm, isa=isa) except KeyboardInterrupt: fm.close() marked_asm_path.unlink() print(". ", end="", flush=True) else: # use maked assembly and extract asm_block and pointer_increment with open(marked_asm_path) as f: marked_asm = f.read() m = re.search(r"pointer_increment=([0-9]+)", marked_asm) if m: pointer_increment = int(m.group(1)) else: os.unlink(marked_asm_path) raise ValueError( "Could not find `pointer_increment=`. Plase place into file." ) print("! ", end="", flush=True) # Compile marked assembly to object for IACA marked_obj = Path(asm_path).with_suffix(".marked.o") if not marked_obj.exists(): check_call([compiler] + cflags + ["-c", str(marked_asm_path), "-o", str(marked_obj)]) return str(marked_asm_path), str(marked_obj), pointer_increment, overwrite def build_kernel( kernel, architecture, compiler, cflags, cflags_name, overwrite=False, dontbuild=False, ): build_path = f"build/{architecture}/{compiler}/{cflags_name}" kernel_assembly = f"{build_path}/{kernel}.s" kernel_object = f"{build_path}/{kernel}.o" executable = f"{build_path}/{kernel}" Path(build_path).mkdir(parents=True, exist_ok=True) if not overwrite: # Overwrite if any kernel specific file is missing overwrite = ( not os.path.exists(kernel_object) or not os.path.exists(kernel_assembly) or not os.path.exists(executable) ) if dontbuild and overwrite: raise ValueError("Must build, but not allowed.") if not Path(f"{build_path}/dummy.o").exists(): check_call([compiler] + cflags + ["-c", "kernels/dummy.c", "-o", f"{build_path}/dummy.o"]) if not Path(f"{build_path}/compiler_version").exists(): # Document compiler version with open(f"{build_path}/compiler_version", "w") as f: f.write(check_output([compiler, "--version"], encoding="utf8", stderr=STDOUT)) if overwrite: # build object + assembly check_call([compiler] + cflags + ["-c", f"kernels/{kernel}.c", "-o", kernel_object]) check_call( [compiler] + cflags + ["-c", f"kernels/{kernel}.c", "-S", "-o", kernel_assembly] ) # build main and link executable executable_cflags = [ os.environ["LIKWID_DEFINES"], os.environ["LIKWID_INC"], os.environ["LIKWID_LIB"], ] + ["-Ofast"] check_call( [compiler] + executable_cflags + [ f"{build_path}/dummy.o", kernel_object, "-DMAIN", f"kernels/{kernel}.c", "-llikwid", "-o", executable, ] ) print(". ", end="", flush=True) else: print("! ", end="", flush=True) return kernel_assembly, executable, overwrite def perfctr(cmd, cores, group="MEM", code_markers=True, verbose=0): """ Run *cmd* with likwid-perfctr and returns result as dict. *group* may be a performance group known to likwid-perfctr or an event string. if CLI argument cores > 1, running with multi-core, otherwise single-core """ # Making sure likwid-perfctr is available: if benchmark.find_executable("likwid-perfctr") is None: print( "likwid-perfctr was not found. Make sure likwid is installed and found in PATH.", file=sys.stderr, ) sys.exit(1) # FIXME currently only single core measurements support! perf_cmd = ["likwid-perfctr", "-f", "-O", "-g", group] cpu = "S0:0" if cores > 1: cpu += "-" + str(cores - 1) # Pinned and measured on cpu perf_cmd += ["-C", cpu] # code must be marked using likwid markers perf_cmd.append("-m") perf_cmd += cmd if verbose > 1: print(" ".join(perf_cmd)) try: with benchmark.fix_env_variable("OMP_NUM_THREADS", None): output = check_output(perf_cmd).decode("utf-8").split("\n") except CalledProcessError as e: print("Executing benchmark failed: {!s}".format(e), file=sys.stderr) sys.exit(1) # TODO multicore output is different and needs to be considered here! results = {} cur_region_name = None cur_region_data = {} for line in output: if line == "STRUCT,Info,3" and cur_region_name is not None: results[cur_region_name] = cur_region_data cur_region_name = None cur_region_data = {} m = re.match(r"TABLE,Region ([a-z\-0-9_]+),", line) if m: cur_region_name = m.group(1) line = line.split(",") try: # Metrics cur_region_data[line[0]] = float(line[1]) continue except ValueError: # Would not convert to float pass except IndexError: # Not a parable line (did not contain any commas) continue try: # Event counters if line[2] == "-" or line[2] == "nan": counter_value = 0 else: counter_value = int(line[2]) if re.fullmatch(r"[A-Z0-9_]+", line[0]) and re.fullmatch( r"[A-Z0-9]+(:[A-Z0-9]+=[0-9A-Fa-fx]+)*", line[1] ): cur_region_data.setdefault(line[0], {}) cur_region_data[line[0]][line[1]] = counter_value continue except (IndexError, ValueError): pass if line[0].endswith(":") and len(line) == 3 and line[2] == "": # CPU information strings cur_region_data[line[0]] = line[1] continue results[cur_region_name] = cur_region_data return results, output def remove_html_tags(text): return re.sub("<.*?>", "", text) def get_intel_style_code(marked_objfile): # Disassembl with Intel syntax cmd = ( "objdump -d --demangle --no-leading-addr --no-leading-headers --no-show-raw-insn " "--x86-asm-syntax=intel" ).split(" ") + [marked_objfile] asm_raw = check_output(cmd).decode() asm_raw = "\n".join([line.strip() for line in asm_raw.split("\n")]) kernel_raw = asm_raw[ asm_raw.index("mov\tebx, 111\nnop") + len("mov\tebx, 111\nnop") : asm_raw.index("mov\tebx, 222\nnop") ] kernel_lines = kernel_raw.split("\n") # Ignore label and jump return "\n".join(kernel_lines[:-2]) def get_ithemal_prediction(code, model="skl"): url = "http://3.18.198.23/predict" assert model in ["skl", "hsw", "ivb"] r = requests.post(url, {"code": code, "model": model}) raw_text = remove_html_tags(r.text) m = re.search("Could not generate a prediction: (.*)", raw_text) if m: print(" error:", m.group(1).strip(), end=" ") return float("nan") m = re.search("Prediction: ([0-9.]+) cycles per iteration", raw_text) if m: return float(m.group(1)) else: return float("nan") def main(): # Check for correct LLVM-MCA version try: llvm_mca = "LLVM version" in check_output(["llvm-mca", "-version"]).decode() except FileNotFoundError: llvm_mca = False build_mark_run_all_kernels( measurements="--no-measurements" not in sys.argv, iaca=False, osaca=True, llvm_mca=llvm_mca ) sys.exit() if __name__ == "__main__": main()