4 # Copyright(c) 2012-2018 Intel Corporation
5 # SPDX-License-Identifier: BSD-3-Clause-Clear
16 def run_command(args
, verbose
=True):
17 result
= subprocess
.run(" ".join(args
), shell
=True,
18 stdout
=subprocess
.PIPE
, stderr
=subprocess
.PIPE
)
19 result
.stdout
= result
.stdout
.decode("ASCII", errors
='ignore')
20 result
.stderr
= result
.stderr
.decode("ASCII", errors
='ignore')
27 # This script purpose is to remove unused functions definitions
28 # It is giving the opportunity to unit test all functions from OCF.
29 # As a parameter should be given path to file containing function,
30 # which is target of testing. However that file has to be after
33 # Output file of this script is not ready to make it. Before that,
34 # has to be given definitions of functions, which are used by
37 # In brief: this script allow wrapping all function calls in UT
40 class UnitTestsSourcesGenerator(object):
41 script_file_abs_path
= ""
42 script_dir_abs_path
= ""
50 test_catalogues_list
= []
51 dirs_to_include_list
= []
53 tests_internal_includes_list
= []
54 framework_includes
= []
56 dirs_with_tests_list
= []
57 test_files_paths_list
= []
59 tested_files_paths_list
= []
61 includes_to_copy_dict
= {}
63 preprocessing_repo
= ""
64 sources_to_test_repo
= ""
67 self
.script_file_abs_path
= os
.path
.realpath(__file__
)
68 self
.script_dir_abs_path
= os
.path
.normpath(
69 os
.path
.dirname(self
.script_file_abs_path
) + os
.sep
)
73 self
.set_main_UT_dir()
74 self
.set_main_env_dir()
75 self
.set_main_tested_dir()
77 self
.test_catalogues_list
= tests_config
.DIRECTORIES_WITH_TESTS_LIST
78 self
.set_includes_to_copy_dict(tests_config
.INCLUDES_TO_COPY_DICT
)
79 self
.set_dirs_to_include()
81 self
.set_tests_internal_includes_list()
82 self
.set_framework_includes()
83 self
.set_files_with_tests_list()
84 self
.set_tested_files_paths_list()
86 self
.set_preprocessing_repo()
87 self
.set_sources_to_test_repo()
89 def preprocessing(self
):
90 tested_files_list
= self
.get_tested_files_paths_list()
91 project_includes
= self
.get_dirs_to_include_list()
92 framework_includes
= self
.get_tests_internal_includes_list()
94 gcc_flags
= " -fno-inline -Dstatic= -Dinline= -E "
95 gcc_command_template
= "gcc "
96 for path
in project_includes
:
97 gcc_command_template
+= " -I " + path
+ " "
99 for path
in framework_includes
:
100 gcc_command_template
+= " -I " + path
102 gcc_command_template
+= gcc_flags
104 for path
in tested_files_list
:
105 preprocessing_dst
= self
.get_preprocessing_repo() \
106 + self
.get_relative_path(path
, self
.get_main_tested_dir())
107 preprocessing_dst_dir
= os
.path
.dirname(preprocessing_dst
)
108 self
.create_dir_if_not_exist(preprocessing_dst_dir
)
110 gcc_command
= gcc_command_template
+ path
+ " > " + preprocessing_dst
112 result
= run_command([gcc_command
])
114 if result
.returncode
!= 0:
115 print(f
"Generating preprocessing for {self.get_main_tested_dir() + path} failed!")
117 run_command(["rm", "-f", preprocessing_dst
])
120 self
.remove_hashes(preprocessing_dst
)
122 print(f
"Preprocessed file {path} saved to {preprocessing_dst}")
124 def copy_includes(self
):
125 includes_dict
= self
.get_includes_to_copy_dict()
127 for dst
, src
in includes_dict
.items():
128 src_path
= os
.path
.normpath(self
.get_main_tested_dir() + src
)
129 if not os
.path
.isdir(src_path
):
130 print(f
"Directory {src_path} given to include does not exists!")
132 dst_path
= os
.path
.normpath(self
.get_main_UT_dir() + dst
)
134 shutil
.rmtree(dst_path
, ignore_errors
=True)
135 shutil
.copytree(src_path
, dst_path
)
137 def get_user_wraps(self
, path
):
138 functions_list
= self
.get_functions_list(path
)
139 functions_list
= [re
.sub(r
'__wrap_([\S]+)\s*[\d]+', r
'\1', line
)
140 for line
in functions_list
if re
.search("__wrap_", line
)]
142 return functions_list
144 def get_autowrap_file_path(self
, test_file_path
):
145 wrap_file_path
= test_file_path
.rsplit('.', 1)[0]
146 wrap_file_path
= wrap_file_path
+ "_generated_wraps.c"
147 return wrap_file_path
149 def prepare_autowraps(self
, test_file_path
, preprocessed_file_path
):
150 functions_to_wrap
= self
.get_functions_calls(
151 self
.get_sources_to_test_repo() + test_file_path
)
152 user_wraps
= set(self
.get_user_wraps(self
.get_main_UT_dir() + test_file_path
))
154 functions_to_wrap
= functions_to_wrap
- user_wraps
156 tags_list
= self
.get_functions_list(preprocessed_file_path
, prototypes
=True)
160 with
open(preprocessed_file_path
) as f
:
162 for function
in functions_to_wrap
:
163 if function
.startswith("env_") or function
.startswith("bug") \
164 or function
.startswith("memcpy"):
165 # added memcpy function to list of ignored functions
166 # because this is macro
168 for tag
in tags_list
:
170 name
, line
= tag
.split()
173 wrap_list
.append(self
.get_function_wrap(code
, line
))
176 wrap_file_path
= self
.get_main_UT_dir() + self
.get_autowrap_file_path(test_file_path
)
178 with
open(wrap_file_path
, "w") as f
:
179 f
.write("/* This file is generated by UT framework */\n")
180 for wrap
in wrap_list
:
183 def prepare_sources_for_testing(self
):
184 test_files_paths
= self
.get_files_with_tests_list()
186 for test_path
in test_files_paths
:
187 path
= self
.get_tested_file_path(self
.get_main_UT_dir() + test_path
)
189 preprocessed_tested_path
= self
.get_preprocessing_repo() + path
190 if not os
.path
.isfile(preprocessed_tested_path
):
191 print(f
"No preprocessed path for {test_path} test file.")
194 tested_src
= self
.get_src_to_test(test_path
, preprocessed_tested_path
)
196 self
.create_dir_if_not_exist(
197 self
.get_sources_to_test_repo() + os
.path
.dirname(test_path
))
199 with
open(self
.get_sources_to_test_repo() + test_path
, "w") as f
:
200 f
.writelines(tested_src
)
202 f
"Sources for {test_path} saved in + \
203 {self.get_sources_to_test_repo() + test_path}")
205 self
.prepare_autowraps(test_path
, preprocessed_tested_path
)
207 def create_main_cmake_lists(self
):
208 buf
= "cmake_minimum_required(VERSION 2.6.0)\n\n"
209 buf
+= "project(OCF_unit_tests C)\n\n"
211 buf
+= "enable_testing()\n\n"
213 buf
+= "include_directories(\n"
214 dirs_to_inc
= self
.get_dirs_to_include_list() + self
.get_framework_includes() \
215 + self
.get_tests_internal_includes_list()
216 for path
in dirs_to_inc
:
217 buf
+= "\t" + path
+ "\n"
220 includes
= self
.get_tests_internal_includes_list()
221 for path
in includes
:
222 buf
+= "\nadd_subdirectory(" + path
+ ")"
225 test_files
= self
.get_files_with_tests_list()
226 test_dirs_to_include
= [os
.path
.dirname(path
) for path
in test_files
]
228 test_dirs_to_include
= self
.remove_duplicates_from_list(test_dirs_to_include
)
230 for path
in test_dirs_to_include
:
231 buf
+= "\nadd_subdirectory(" + self
.get_sources_to_test_repo() + path
+ ")"
233 with
open(self
.get_main_UT_dir() + "CMakeLists.txt", "w") as f
:
236 print(f
"Main CMakeLists.txt generated written to {self.get_main_UT_dir()} CMakeLists.txt")
238 def generate_cmakes_for_tests(self
):
239 test_files_paths
= self
.get_files_with_tests_list()
241 for test_path
in test_files_paths
:
242 tested_file_path
= self
.get_sources_to_test_repo() + test_path
243 if not os
.path
.isfile(tested_file_path
):
244 print(f
"No source to test for {test_path} test")
247 test_file_path
= self
.get_main_UT_dir() + test_path
249 cmake_buf
= self
.generate_test_cmake_buf(test_file_path
, tested_file_path
)
251 cmake_path
= self
.get_sources_to_test_repo() + test_path
252 cmake_path
= os
.path
.splitext(cmake_path
)[0] + ".cmake"
253 with
open(cmake_path
, "w") as f
:
254 f
.writelines(cmake_buf
)
255 print(f
"cmake file for {test_path} written to {cmake_path}")
257 cmake_lists_path
= os
.path
.dirname(cmake_path
) + os
.sep
258 self
.update_cmakelists(cmake_lists_path
, cmake_path
)
260 def generate_test_cmake_buf(self
, test_file_path
, tested_file_path
):
261 test_file_name
= os
.path
.basename(test_file_path
)
262 target_name
= os
.path
.splitext(test_file_name
)[0]
264 add_executable
= "add_executable(" + target_name
+ " " + test_file_path
+ " " + \
265 tested_file_path
+ ")\n"
267 libraries
= "target_link_libraries(" + target_name
+ " libcmocka.so ocf_env)\n"
269 add_test
= "add_test(" + target_name
+ " ${CMAKE_CURRENT_BINARY_DIR}/" + target_name
+ ")\n"
271 tgt_properties
= "set_target_properties(" + target_name
+ "\n" + \
273 "COMPILE_FLAGS \"-fno-inline -Dstatic= -Dinline= -w \"\n"
275 link_flags
= self
.generate_cmake_link_flags(test_file_path
)
276 tgt_properties
+= link_flags
+ ")"
278 buf
= add_executable
+ libraries
+ add_test
+ tgt_properties
282 def generate_cmake_link_flags(self
, path
):
285 autowraps_path
= self
.get_autowrap_file_path(path
)
286 functions_to_wrap
= self
.get_functions_to_wrap(path
)
287 functions_to_wrap
+= self
.get_functions_to_wrap(autowraps_path
)
289 for function_name
in functions_to_wrap
:
290 ret
+= ",--wrap=" + function_name
292 ret
= "LINK_FLAGS \"-Wl" + ret
+ "\"\n"
296 def update_cmakelists(self
, cmake_lists_path
, cmake_name
):
297 with
open(cmake_lists_path
+ "CMakeLists.txt", "a+") as f
:
298 f
.seek(0, os
.SEEK_SET
)
299 new_line
= "include(" + os
.path
.basename(cmake_name
) + ")\n"
301 if new_line
not in f
.read():
304 def get_functions_to_wrap(self
, path
):
305 functions_list
= self
.get_functions_list(path
)
306 functions_list
= [re
.sub(r
'__wrap_([\S]+)\s*[\d]+', r
'\1', line
) for line
in functions_list
307 if re
.search("__wrap_", line
)]
309 return functions_list
311 def get_functions_to_leave(self
, path
):
312 with
open(path
) as f
:
313 lines
= f
.readlines()
316 tags_pattern
= re
.compile(r
"<functions_to_leave>[\s\S]*</functions_to_leave>")
318 buf
= re
.findall(tags_pattern
, buf
)
324 buf
= re
.sub(r
'<.*>', '', buf
)
325 buf
= re
.sub(r
'[^a-zA-Z0-9_\n]+', '', buf
)
327 ret
= buf
.split("\n")
328 ret
= [name
for name
in ret
if name
]
331 def get_functions_list(self
, file_path
, prototypes
=None):
332 ctags_path
= self
.get_ctags_path()
334 ctags_args
= "--c-types=f"
336 ctags_args
+= " --c-kinds=+p"
337 # find all functions' definitions | put tabs instead of spaces |
338 # take only columns with function name and line number | sort in descending order
339 result
= run_command([ctags_path
, "-x", ctags_args
, file_path
,
340 "--language-force=c | sed \"s/ \\+/\t/g\" | cut -f 1,3 | sort -nsr "
343 # 'output' is string, but it has to be changed to list
344 output
= list(filter(None, result
.stdout
.split("\n")))
347 def remove_functions_from_list(self
, functions_list
, to_remove_list
):
348 ret
= functions_list
[:]
349 for function_name
in to_remove_list
:
350 ret
= [line
for line
in ret
if not re
.search(r
'\b%s\b' % function_name
, line
)]
353 def get_src_to_test(self
, test_path
, preprocessed_tested_path
):
354 functions_to_leave
= self
.get_functions_to_leave(self
.get_main_UT_dir() + test_path
)
356 functions_to_leave
.append(self
.get_tested_function_name(self
.get_main_UT_dir() + test_path
))
357 functions_list
= self
.get_functions_list(preprocessed_tested_path
)
359 functions_list
= self
.remove_functions_from_list(functions_list
, functions_to_leave
)
361 with
open(preprocessed_tested_path
) as f
:
363 for function
in functions_list
:
364 line
= function
.split("\t")[1]
367 self
.remove_function_body(ret
, line
)
371 def set_tested_files_paths_list(self
):
372 test_files_list
= self
.get_files_with_tests_list()
374 for f
in test_files_list
:
375 self
.tested_files_paths_list
.append(self
.get_main_tested_dir()
376 + self
.get_tested_file_path(
377 self
.get_main_UT_dir() + f
))
379 self
.tested_files_paths_list
= self
.remove_duplicates_from_list(
380 self
.tested_files_paths_list
)
382 def get_tested_files_paths_list(self
):
383 return self
.tested_files_paths_list
385 def get_files_with_tests_list(self
):
386 return self
.test_files_paths_list
388 def set_files_with_tests_list(self
):
389 test_catalogues_list
= self
.get_tests_catalouges_list()
390 for catalogue
in test_catalogues_list
:
391 dir_with_tests_path
= self
.get_main_UT_dir() + catalogue
393 for path
, dirs
, files
in os
.walk(dir_with_tests_path
):
394 test_files
= self
.get_test_files_from_dir(path
+ os
.sep
)
396 for test_file_name
in test_files
:
397 test_rel_path
= os
.path
.relpath(path
+ os
.sep
+ test_file_name
,
398 self
.get_main_UT_dir())
399 self
.test_files_paths_list
.append(test_rel_path
)
401 def are_markups_valid(self
, path
):
402 file_path
= self
.get_tested_file_path(path
)
403 function_name
= self
.get_tested_function_name(path
)
405 if file_path
is None:
406 print(f
"{path} file has no tested_file tag!")
408 elif not os
.path
.isfile(self
.get_main_tested_dir() + file_path
):
409 print(f
"Tested file given in {path} does not exist!")
412 if function_name
is None:
413 print(f
"{path} file has no tested_function_name tag!")
418 def create_dir_if_not_exist(self
, path
):
419 if not os
.path
.isdir(path
):
427 def get_tested_file_path(self
, test_file_path
):
428 with
open(test_file_path
) as f
:
432 tags_pattern
= re
.compile(r
"<tested_file_path>[\s\S]*</tested_file_path>")
433 buf
= re
.findall(tags_pattern
, buf
)
440 buf
= re
.sub(r
'<[^>]*>', '', buf
)
441 buf
= re
.sub(r
'\s+', '', buf
)
448 def get_tested_function_name(self
, test_file_path
):
449 with
open(test_file_path
) as f
:
453 tags_pattern
= re
.compile(r
"<tested_function>[\s\S]*</tested_function>")
454 buf
= re
.findall(tags_pattern
, buf
)
461 buf
= re
.sub(r
'<[^>]*>', '', buf
)
462 buf
= re
.sub('//', '', buf
)
463 buf
= re
.sub(r
'\s+', '', buf
)
470 def get_test_files_from_dir(self
, path
):
471 ret
= os
.listdir(path
)
472 ret
= [name
for name
in ret
if os
.path
.isfile(path
+ os
.sep
+ name
)
473 and (name
.endswith(".c") or name
.endswith(".h"))]
474 ret
= [name
for name
in ret
if self
.are_markups_valid(path
+ name
)]
478 def get_list_of_directories(self
, path
):
479 if not os
.path
.isdir(path
):
482 ret
= os
.listdir(path
)
483 ret
= [name
for name
in ret
if not os
.path
.isfile(path
+ os
.sep
+ name
)]
484 ret
= [os
.path
.normpath(name
) + os
.sep
for name
in ret
]
488 def remove_hashes(self
, path
):
489 with
open(path
) as f
:
492 buf
= [l
for l
in buf
if not re
.search(r
'.*#.*', l
)]
494 with
open(path
, "w") as f
:
498 for i
in range(len(padding
)):
500 padding
[i
] = padding
[i
].split("#")[0]
505 f
.writelines(padding
)
508 def find_function_end(self
, code_lines_list
, first_line_of_function_index
):
510 current_line_index
= first_line_of_function_index
513 if "{" in code_lines_list
[current_line_index
]:
514 brackets_counter
+= code_lines_list
[current_line_index
].count("{")
515 brackets_counter
-= code_lines_list
[current_line_index
].count("}")
518 current_line_index
+= 1
520 while brackets_counter
> 0:
521 current_line_index
+= 1
522 if "{" in code_lines_list
[current_line_index
]:
523 brackets_counter
+= code_lines_list
[current_line_index
].count("{")
524 brackets_counter
-= code_lines_list
[current_line_index
].count("}")
525 elif "}" in code_lines_list
[current_line_index
]:
526 brackets_counter
-= code_lines_list
[current_line_index
].count("}")
528 return current_line_index
530 def get_functions_calls(self
, file_to_compile
):
531 out_dir
= "/tmp/ocf_ut"
532 out_file
= out_dir
+ "/ocf_obj.o"
533 self
.create_dir_if_not_exist(out_dir
)
534 cmd
= "/usr/bin/gcc -o " + out_file
+ " -c " + file_to_compile
+ " &> /dev/null"
535 run_command([cmd
], verbose
=None)
536 result
= run_command(["/usr/bin/nm -u " + out_file
+ " | cut -f2 -d\'U\'"])
537 return set(result
.stdout
.split())
539 def remove_function_body(self
, code_lines_list
, line_id
):
541 while "{" not in code_lines_list
[line_id
]:
542 if ";" in code_lines_list
[line_id
]:
548 last_line_id
= self
.find_function_end(code_lines_list
, line_id
)
550 code_lines_list
[line_id
] = code_lines_list
[line_id
].split("{")[0]
551 code_lines_list
[line_id
] += ";"
553 del code_lines_list
[line_id
+ 1: last_line_id
+ 1]
555 def get_function_wrap(self
, code_lines_list
, line_id
):
557 # Line numbering starts with one, list indexing with zero
560 # If returned type is not present, it should be in line above
562 code_lines_list
[line_id
].split("(")[0].rsplit()[1]
567 ret
.append(code_lines_list
[line_id
])
568 if ")" in code_lines_list
[line_id
]:
572 # Tags list contains both prototypes and definitions, here we recoginze
573 # with which one we deals
576 if "{" in ret
[-1] or "{" in ret
[-2]:
583 ret
[-1] = ret
[-1].split(delimiter
)[0]
589 function_name
= ret
[line_with_name
].split("(")[0].rsplit(maxsplit
=1)[1]
592 function_name
= ret
[line_with_name
].split("(")[0]
594 function_new_name
= "__wrap_" + function_name
.replace("*", "")
595 ret
[0] = ret
[0].replace(function_name
, function_new_name
)
599 def set_ctags_path(self
):
600 result
= run_command(["/usr/bin/ctags --version &> /dev/null"])
601 if result
.returncode
== 0:
602 path
= "/usr/bin/ctags "
603 result
= run_command([path
, "--c-types=f"], verbose
=None)
604 if not re
.search("unrecognized option", result
.stdout
, re
.IGNORECASE
):
605 self
.ctags_path
= path
608 result
= run_command(["/usr/local/bin/ctags --version &> /dev/null"])
609 if result
.returncode
== 0:
610 path
= "/usr/local/bin/ctags "
611 result
= run_command(["path", "--c-types=f"], verbose
=None)
612 if not re
.search("unrecognized option", result
.stdout
, re
.IGNORECASE
):
613 self
.ctags_path
= path
616 print("ERROR: Current ctags version don't support \"--c-types=f\" parameter!")
619 def get_ctags_path(self
):
620 return self
.ctags_path
622 def get_tests_catalouges_list(self
):
623 return self
.test_catalogues_list
625 def get_relative_path(self
, original_path
, part_to_remove
):
626 return original_path
.split(part_to_remove
, 1)[1]
628 def get_dirs_to_include_list(self
):
629 return self
.dirs_to_include_list
631 def set_dirs_to_include(self
):
632 self
.dirs_to_include_list
= [self
.get_main_tested_dir() + name
634 tests_config
.DIRECTORIES_TO_INCLUDE_FROM_PROJECT_LIST
]
636 def set_tests_internal_includes_list(self
):
637 self
.tests_internal_includes_list
= [self
.get_main_UT_dir() + name
639 tests_config
.DIRECTORIES_TO_INCLUDE_FROM_UT_LIST
]
641 def set_preprocessing_repo(self
):
642 self
.preprocessing_repo
= self
.get_main_UT_dir() \
643 + tests_config
.PREPROCESSED_SOURCES_REPOSITORY
645 def set_sources_to_test_repo(self
):
646 self
.sources_to_test_repo
= self
.get_main_UT_dir() + tests_config
.SOURCES_TO_TEST_REPOSITORY
648 def get_sources_to_test_repo(self
):
649 return self
.sources_to_test_repo
651 def get_preprocessing_repo(self
):
652 return self
.preprocessing_repo
654 def get_tests_internal_includes_list(self
):
655 return self
.tests_internal_includes_list
657 def get_script_dir_path(self
):
658 return os
.path
.normpath(self
.script_dir_abs_path
) + os
.sep
660 def get_main_UT_dir(self
):
661 return os
.path
.normpath(self
.main_UT_dir
) + os
.sep
663 def get_main_env_dir(self
):
664 return os
.path
.normpath(self
.main_env_dir
) + os
.sep
666 def get_main_tested_dir(self
):
667 return os
.path
.normpath(self
.main_tested_dir
) + os
.sep
669 def remove_duplicates_from_list(self
, l
):
672 def set_framework_includes(self
):
673 self
.framework_includes
= tests_config
.FRAMEWORK_DIRECTORIES_TO_INCLUDE_LIST
675 def get_framework_includes(self
):
676 return self
.framework_includes
678 def set_includes_to_copy_dict(self
, files_to_copy_dict
):
679 self
.includes_to_copy_dict
= files_to_copy_dict
681 def get_includes_to_copy_dict(self
):
682 return self
.includes_to_copy_dict
684 def set_main_env_dir(self
):
685 main_env_dir
= os
.path
.normpath(os
.path
.normpath(self
.get_script_dir_path()
688 MAIN_DIRECTORY_OF_ENV_FILES
))
689 if not os
.path
.isdir(main_env_dir
):
690 print("Given path to main env directory is wrong!")
693 self
.main_env_dir
= main_env_dir
695 def set_main_UT_dir(self
):
696 main_UT_dir
= os
.path
.normpath(os
.path
.normpath(self
.get_script_dir_path()
699 MAIN_DIRECTORY_OF_UNIT_TESTS
))
700 if not os
.path
.isdir(main_UT_dir
):
701 print("Given path to main UT directory is wrong!")
704 self
.main_UT_dir
= main_UT_dir
706 def set_main_tested_dir(self
):
707 main_tested_dir
= os
.path
.normpath(os
.path
.normpath(self
.get_script_dir_path()
710 MAIN_DIRECTORY_OF_TESTED_PROJECT
))
711 if not os
.path
.isdir(main_tested_dir
):
712 print("Given path to main tested directory is wrong!")
715 self
.main_tested_dir
= main_tested_dir
719 generator
= UnitTestsSourcesGenerator()
720 generator
.copy_includes()
721 generator
.preprocessing()
722 generator
.prepare_sources_for_testing()
723 generator
.create_main_cmake_lists()
724 generator
.generate_cmakes_for_tests()
726 print("Files for testing generated!")
729 if __name__
== "__main__":