import os from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, FIRST_COMPLETED, as_completed import time import re pool = ThreadPoolExecutor(max_workers=96) TIMEOUT = 2000 work_dir = "." data_dir = "./benchmark" data_suffix = ".bench" res_dir = "exp_result" class ExitStatus: normal = 0 timeout = 1 runtime_error = 2 def run_shell_command(command, output_file, timeout): res = 0 T1 = time.time() if timeout: res = os.system("timeout %ds stdbuf -oL %s > %s 2>&1 " % ( timeout, command, output_file )) else: res = os.system("%s > %s 2>&1 " % (command, output_file)) T2 = time.time() exec_time = T2 - T1 exit_status = ExitStatus.normal # if(res == 31744): # exit_status = ExitStatus.timeout # if(res != 0): # exit_status = ExitStatus.runtime_error return (exit_status, exec_time) class AtalantaSolver: name = "Atalanta" def run(filename, output_file, timeout): return run_shell_command("./Atalanta/atalanta -b 10000 %s" % filename, output_file, timeout) def analyse(output_file): content = open(output_file, "r").read() p1 = re.compile(r'Fault coverage\s*:\s*(\d+\.\d+)\s*%', re.S) coverage = p1.findall(content)[0] p2 = re.compile(r'Number of test patterns before compaction\s*:\s*(\d+)', re.S) cube = p2.findall(content)[0] p3 = re.compile(r'Number of test patterns after compaction\s*:\s*(\d+)', re.S) pattern = p3.findall(content)[0] return (coverage, cube, pattern) class ATPGLS: name = "ATPG-LS" def run(filename, output_file, timeout): return run_shell_command("./crun %s" % filename, output_file, timeout) def analyse(output_file): print("anal:" + output_file) content = open(output_file, "r").read() p1 = re.compile(r'coverage\s*:\s*(\d+\.\d+)\s*%', re.S) if len(p1.findall(content)) == 0: return("ERROR", "ERROR", "ERROR") coverage = p1.findall(content)[-1] p2 = re.compile(r'pattern\s*:\s*(\d+)', re.S) cube = p2.findall(content)[-1] p3 = re.compile(r'pattern\s*:\s*(\d+)', re.S) pattern = p3.findall(content)[-1] return (coverage, cube, pattern) class TGPro: name = "TGPro" def run(filename, output_file, timeout): return run_shell_command("./tg-pro/bin/atpgSat %s" % filename, output_file, timeout) def analyse(output_file): content = open(output_file, "r").read() total = re.compile(r'Total: (\d+)', re.S).findall(content)[1] detectable = re.compile(r'Detectable: (\d+)', re.S).findall(content)[0] undetectable = re.compile(r'Undetectable: (\d+)', re.S).findall(content)[0] print(detectable, undetectable, total) coverage = str((int(undetectable) + int(detectable)) / int(total) * 100) + "%" cube = "-*-" pattern = "-*-" return (coverage, cube, pattern) class OpenTPG: name = "OpenTPG" def run(filename, output_file, timeout): (path, filename) = os.path.split(filename) fault_file = os.path.join(res_dir,"%s_%s.txt" % (solver.name, filename)) cube_file = os.path.join(res_dir,"%s_%s.txt" % (solver.name, filename)) return run_shell_command("./OpenTPG/opentpg/opentpg -n %s -f %s -c /dev/null -u %s" \ % (filename, output_file), output_file, timeout) def analyse(output_file): content = open(output_file, "r").read() p1 = re.compile(r'coverage:\s*:\s*(\d+\.\d+)\s*%', re.S) coverage = p1.findall(content)[-1] p2 = re.compile(r'pattern:\s*:\s*(\d+)', re.S) cube = p2.findall(content)[-1] p3 = re.compile(r'pattern:\s*:\s*(\d+)', re.S) pattern = p3.findall(content)[-1] return (coverage, cube, pattern) class Table: def __init__(self, header): self.header = header self.col_size = len(header) self.lines = [] def add_line(self, cols): if len(cols) != self.col_size: raise Exception("cols number error! need %d but you have %d." % (self.col_size, len(cols))) for var in cols: var = str(var) self.lines.append(cols) def print_table(self): col_widths = [] for var in self.header: col_widths.append(len(var)) for i in range(self.col_size): for j in range(len(self.lines)): self.lines[j][i] = str(self.lines[j][i]) for i in range(self.col_size): for line in self.lines: col_widths[i] = max(col_widths[i], len(line[i])) table_with = 1 for width in col_widths: table_with = table_with + width + 3 for i in range(table_with): print("-", end="") print() def print_with_filler(str, width, filler): print(str, end="") for i in range(len(str), width): print(filler, end="") print("| ", end="") for i in range(self.col_size): print_with_filler(self.header[i], col_widths[i], " ") print(" | ", end="") print() print("| ", end="") for i in range(self.col_size): print_with_filler("", col_widths[i], "-") print(" | ", end="") print() for line in self.lines: print("| ", end="") for i in range(self.col_size): print_with_filler(line[i], col_widths[i], " ") print(" | ", end="") print() for i in range(table_with): print("-", end="") print() class Logger: def __init__(self, filename): self.f = open(filename, "w") def log(self, str): print(str) self.f.write(str + "\n") self.f.flush() table = None def multiprocess_run_solver(solver, input_file): (path, filename) = os.path.split(input_file) out_file = os.path.join(res_dir,"%s_%s.txt" % (solver.name, filename)) (status, time) = solver.run(input_file, out_file, TIMEOUT) # time = "-*-" # status = ExitStatus.normal fault = "-*-" cube = "-*-" pattern = "-*-" if status == ExitStatus.runtime_error: fault = "Runtime Error" elif status == ExitStatus.timeout: fault = "Time Out" else: (f, c, p) = solver.analyse(out_file) fault = f cube = c pattern = p table.add_line([input_file, fault, time, cube, pattern]) return input_file if __name__ == '__main__': logger = Logger("check.log") os.chdir(work_dir) os.makedirs(res_dir, exist_ok=True) all_task = [] solver = ATPGLS table = Table(["data", "fault coverage(%s)" % solver.name, "time(%s)" % solver.name, "cube(%s)" % solver.name, "pattern(%s)" % solver.name ]) for filename in os.listdir(data_dir): if not filename.endswith(data_suffix): continue all_task.append(pool.submit(multiprocess_run_solver, solver, os.path.join(data_dir, filename))) s = 0 for task in as_completed(all_task): data = task.result() s = s + 1 logger.log("[{}/{}] 任务 {}".format(s, len(all_task), data)) table.print_table()