First draft for multithreaded Modelsim simulation
This commit is contained in:
Ganesh Gore
parent
f05aede868
commit
373dbe0718
|
|
@ -3,6 +3,9 @@ import sys
|
|||
import os
|
||||
import re
|
||||
import glob
|
||||
import time
|
||||
import threading
|
||||
from datetime import timedelta
|
||||
import argparse
|
||||
import subprocess
|
||||
import logging
|
||||
|
|
@ -11,6 +14,7 @@ from configparser import ConfigParser, ExtendedInterpolation
|
|||
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
|
||||
# Configure logging system
|
||||
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
|
||||
FILE_LOG_FORMAT = '%(levelname)s (%(threadName)10s) - %(message)s'
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout,
|
||||
format='%(levelname)s (%(threadName)10s) - %(message)s')
|
||||
logger = logging.getLogger('Modelsim_run_log')
|
||||
|
|
@ -22,6 +26,11 @@ parser = argparse.ArgumentParser()
|
|||
parser.add_argument('files', nargs='+',
|
||||
help="Pass SimulationDeckInfo generated by OpenFPGA flow" +
|
||||
" or pass taskname <taskname> <run_number[optional]>")
|
||||
parser.add_argument('--maxthreads', type=int, default=2,
|
||||
help="Number of fpga_flow threads to run default = 2," +
|
||||
"Typically <= Number of processors on the system")
|
||||
parser.add_argument('--debug', action="store_true",
|
||||
help="Run script in debug mode")
|
||||
parser.add_argument('--modelsim_proc_tmpl', type=str,
|
||||
help="Modelsim proc template file")
|
||||
parser.add_argument('--modelsim_runsim_tmpl', type=str,
|
||||
|
|
@ -77,7 +86,7 @@ args.modelsim_runsim_tmpl = os.path.abspath(args.modelsim_runsim_tmpl)
|
|||
|
||||
def main():
|
||||
if os.path.isfile(args.files[0]):
|
||||
run_modelsim(args.files)
|
||||
create_tcl_script(args.files)
|
||||
else:
|
||||
# Check if task directory exists and consistent
|
||||
taskname = args.files[0]
|
||||
|
|
@ -92,7 +101,17 @@ def main():
|
|||
if not os.path.isdir(temp_dir):
|
||||
clean_up_and_exit("Task run directory [%s] not found" % temp_dir)
|
||||
|
||||
logfile = os.path.join(gc["task_dir"], taskname, task_run, "*.log")
|
||||
# = = = = = = = Create a current script log file handler = = = =
|
||||
logfile_path = os.path.join(gc["task_dir"],
|
||||
taskname, task_run, "modelsim_run.log")
|
||||
logfilefh = logging.FileHandler(logfile_path, "w")
|
||||
logfilefh.setFormatter(logging.Formatter(FILE_LOG_FORMAT))
|
||||
logger.addHandler(logfilefh)
|
||||
logger.info("Created log file at %s" % logfile_path)
|
||||
# = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =
|
||||
|
||||
# = = = = Read Task log file and extract run directory = = =
|
||||
logfile = os.path.join(gc["task_dir"], taskname, task_run, "*_out.log")
|
||||
logfiles = glob.glob(logfile)
|
||||
if not len(logfiles):
|
||||
clean_up_and_exit("No successful run found in [%s]" % temp_dir)
|
||||
|
|
@ -108,7 +127,7 @@ def main():
|
|||
if os.path.isfile(INIfile):
|
||||
task_ini_files.append(INIfile)
|
||||
logger.info(f"Found {len(task_ini_files)} INI files")
|
||||
run_modelsim(task_ini_files)
|
||||
create_tcl_script(task_ini_files)
|
||||
|
||||
|
||||
def clean_up_and_exit(msg):
|
||||
|
|
@ -117,7 +136,8 @@ def clean_up_and_exit(msg):
|
|||
exit(1)
|
||||
|
||||
|
||||
def run_modelsim(files):
|
||||
def create_tcl_script(files):
|
||||
runsim_files = []
|
||||
for eachFile in files:
|
||||
eachFile = os.path.abspath(eachFile)
|
||||
pDir = os.path.dirname(eachFile)
|
||||
|
|
@ -173,42 +193,78 @@ def run_modelsim(files):
|
|||
with open(proc_filename, 'w', encoding='utf-8') as tclout:
|
||||
tclout.write(open(args.modelsim_proc_tmpl,
|
||||
encoding='utf-8').read())
|
||||
|
||||
# Execute modelsim
|
||||
if args.run_sim:
|
||||
os.chdir(args.modelsim_run_dir)
|
||||
modelsim_run_cmd = ["vsim", "-c", "-do", runsim_filename]
|
||||
out = run_command("ModelSim Run", "modelsim_run.log",
|
||||
modelsim_run_cmd)
|
||||
logger.info(re.findall(r"(.*Errors.*Warning.*)", out))
|
||||
else:
|
||||
logger.info("Created runsim and proc files")
|
||||
logger.info(f"runsim_filename {runsim_filename}")
|
||||
logger.info(f"proc_filename {proc_filename}")
|
||||
runsim_files.append({
|
||||
"modelsim_run_dir": args.modelsim_run_dir,
|
||||
"runsim_filename": runsim_filename,
|
||||
"status" :False,
|
||||
"finished" : True
|
||||
})
|
||||
# Execute modelsim
|
||||
if args.run_sim:
|
||||
thread_sema = threading.Semaphore(args.maxthreads)
|
||||
logger.info("Launching %d parallel threads" % args.maxthreads)
|
||||
thread_list = []
|
||||
for thread_no, eachjob in enumerate(runsim_files):
|
||||
t = threading.Thread(target=run_modelsim_thread,
|
||||
name=f"Thread_{thread_no:d}",
|
||||
args=(thread_sema, eachjob, runsim_files))
|
||||
t.start()
|
||||
thread_list.append(t)
|
||||
for eachthread in thread_list:
|
||||
eachthread.join()
|
||||
exit()
|
||||
else:
|
||||
logger.info("Created runsim and proc files")
|
||||
logger.info(f"runsim_filename {runsim_filename}")
|
||||
logger.info(f"proc_filename {proc_filename}")
|
||||
from pprint import pprint
|
||||
pprint(runsim_files)
|
||||
|
||||
|
||||
def run_command(taskname, logfile, command, exit_if_fail=True):
|
||||
# os.chdir(os.pardir)
|
||||
logger.info("Launching %s " % taskname)
|
||||
with open(logfile, 'w+') as output:
|
||||
def run_modelsim_thread(s, eachJob, job_list):
|
||||
os.chdir(eachJob["modelsim_run_dir"])
|
||||
with s:
|
||||
thread_name = threading.currentThread().getName()
|
||||
eachJob["starttime"] = time.time()
|
||||
try:
|
||||
output.write(os.getcwd() + "\n")
|
||||
output.write(" ".join(command)+"\n")
|
||||
process = subprocess.run(command,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
universal_newlines=True)
|
||||
output.write(process.stdout)
|
||||
if process.returncode:
|
||||
logger.error("%s run failed with returncode %d" %
|
||||
(taskname, process.returncode))
|
||||
except (Exception, subprocess.CalledProcessError) as e:
|
||||
logger.exception("failed to execute %s" % taskname)
|
||||
return None
|
||||
logger.info("%s is written in file %s" % (taskname, logfile))
|
||||
return process.stdout
|
||||
|
||||
logfile = "%s_modelsim.log" % thread_name
|
||||
with open(logfile, 'w+') as output:
|
||||
output.write("* "*20 + '\n')
|
||||
output.write("RunDirectory : %s\n" % os.getcwd())
|
||||
command = ["vsim", "-c", "-do", eachJob["runsim_filename"]]
|
||||
output.write(" ".join(command) + '\n')
|
||||
output.write("* "*20 + '\n')
|
||||
logger.info("Running modelsim with [%s]" % " ".join(command))
|
||||
process = subprocess.Popen(command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True)
|
||||
for line in process.stdout:
|
||||
if "Errors" in line:
|
||||
logger.debug(line.strip())
|
||||
sys.stdout.buffer.flush()
|
||||
output.write(line)
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
raise subprocess.CalledProcessError(0, " ".join(command))
|
||||
eachJob["status"] = True
|
||||
except:
|
||||
logger.exception("Failed to execute openfpga flow - " +
|
||||
eachJob["name"])
|
||||
if not args.continue_on_fail:
|
||||
os._exit(1)
|
||||
eachJob["endtime"] = time.time()
|
||||
timediff = timedelta(seconds=(eachJob["endtime"]-eachJob["starttime"]))
|
||||
timestr = humanize.naturaldelta(timediff) if "humanize" in sys.modules \
|
||||
else str(timediff)
|
||||
logger.info("%s Finished with returncode %d, Time Taken %s " %
|
||||
(thread_name, process.returncode, timestr))
|
||||
eachJob["finished"] = True
|
||||
no_of_finished_job = sum([not eachJ["finished"] for eachJ in job_list])
|
||||
logger.info("***** %d runs pending *****" % (no_of_finished_job))
|
||||
|
||||
if __name__ == "__main__":
|
||||
if args.debug:
|
||||
logger.info("Setting loggger in debug mode")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue