ivampiresp/Spdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: report test domains for tests.

Browse Source 
This will allow us to run test cases under multiple suites without
clogging up the logs too much, but it will also preserve information
about which test suites were run or (more importantly) not run.

Change-Id: I2434a54a0877ae36b9f84bfab8a62653ac1172f8
Signed-off-by: Seth Howell <seth.howell@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/477367
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Karol Latecki <karol.latecki@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
This commit is contained in:
Seth Howell 2019-12-09 14:31:21 -07:00 committed by Tomasz Zawadzki
parent 4e3104d0e9
commit ea781d6dbb
2 changed files with 34 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
autorun_post.py
View File

@ -19,17 +19,17 @@ def highest_value(inp):
def generateTestCompletionTables(output_dir, completion_table):
data_table = pd.DataFrame(completion_table, columns=["Agent", "Test", "With Asan", "With UBsan"])
data_table = pd.DataFrame(completion_table, columns=["Agent", "Domain", "Test", "With Asan", "With UBsan"])
data_table.to_html(os.path.join(output_dir, 'completions_table.html'))
os.makedirs(os.path.join(output_dir, "post_process"), exist_ok=True)
pivot_by_agent = pd.pivot_table(data_table, index=["Agent", "Test"])
pivot_by_agent = pd.pivot_table(data_table, index=["Agent", "Domain", "Test"])
pivot_by_agent.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_agent.html'))
pivot_by_test = pd.pivot_table(data_table, index=["Test", "Agent"])
pivot_by_test = pd.pivot_table(data_table, index=["Domain", "Test", "Agent"])
pivot_by_test.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_test.html'))
pivot_by_asan = pd.pivot_table(data_table, index=["Test"], values=["With Asan"], aggfunc=highest_value)
pivot_by_asan = pd.pivot_table(data_table, index=["Domain", "Test"], values=["With Asan"], aggfunc=highest_value)
pivot_by_asan.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_asan.html'))
pivot_by_ubsan = pd.pivot_table(data_table, index=["Test"], values=["With UBsan"], aggfunc=highest_value)
pivot_by_ubsan = pd.pivot_table(data_table, index=["Domain", "Test"], values=["With UBsan"], aggfunc=highest_value)
pivot_by_ubsan.to_html(os.path.join(output_dir, "post_process", 'completions_table_by_ubsan.html'))
@ -101,12 +101,12 @@ def getCompletions(completionFile, test_list, test_completion_table):
ubsan_enabled = "ubsan" in completions
for line in completions.splitlines():
line = line.strip()
try:
test_list[line] = (True, asan_enabled | test_list[line][1], ubsan_enabled | test_list[line][2])
test_completion_table.append([agent_name, line, asan_enabled, ubsan_enabled])
domain, test_name = line.strip().split()
test_list[test_name] = (True, asan_enabled | test_list[test_name][1], ubsan_enabled | test_list[test_name][2])
test_completion_table.append([agent_name, domain, test_name, asan_enabled, ubsan_enabled])
try:
test_completion_table.remove(["None", line, False, False])
test_completion_table.remove(["None", "None", test_name, False, False])
except ValueError:
continue
except KeyError:
@ -139,8 +139,14 @@ def aggregateCompletedTests(output_dir, repo_dir):
with open(testFiles[0], 'r') as raw_test_list:
for line in raw_test_list:
test_list[line.strip()] = (False, False, False)
test_completion_table.append(["None", line.strip(), False, False])
try:
test_name = line.strip()
except Exception:
print("Failed to parse a test type.")
return 1
test_list[test_name] = (False, False, False)
test_completion_table.append(["None", "None", test_name, False, False])
for completionFile in completionFiles:
getCompletions(completionFile, test_list, test_completion_table)

19
test/common/autotest_common.sh
View File

@ -591,6 +591,12 @@ function run_test() {
local test_name="$1"
shift
if [ -n "$test_domain" ]; then
export test_domain="${test_domain}.${test_name}"
else
export test_domain="$test_name"
fi
timing_enter $test_name
echo "************************************"
echo "START TEST $test_name"
@ -601,9 +607,18 @@ function run_test() {
echo "************************************"
echo "END TEST $test_name"
echo "************************************"
echo "$test_name" >> $output_dir/test_completions.txt
timing_exit $test_name
export test_domain=${test_domain%"$test_name"}
if [ -n "$test_domain" ]; then
export test_domain=${test_domain%?}
fi
if [ -z "$test_domain" ]; then
echo "top_level $test_name" >> $output_dir/test_completions.txt
else
echo "$test_domain $test_name" >> $output_dir/test_completions.txt
fi
xtrace_restore
}