Fixing tests so they become platform-independent

Author: dgarijo

inspect4py/__init__.py

@@ -1 +1 @@
-__version__ = '0.0.6'
+__version__ = '0.0.7'

inspect4py/cli.py

@@ -1259,9 +1259,16 @@ def main(input_path, output_dir, ignore_dir_pattern, ignore_file_pattern, requir
          control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection, readme,
          metadata, data_flow, symbol_table):
     if data_flow:
-        LANGUAGE = Language(symbol_table, "python")
+        if symbol_table == "my_language.so": # default option
+            path_to_languages = str(Path(__file__).parent / "resources")
+            if sys.platform.startswith("win") or sys.platform.startswith("cygwin"):
+                language = Language(path_to_languages + os.path.sep + "python_win.so", "python")
+            else:
+                language = Language(path_to_languages + os.path.sep + "python_unix.so", "python")
+        else:
+            language = Language(symbol_table, "python")
         parser = Parser()
-        parser.set_language(LANGUAGE)
+        parser.set_language(language)
         parser = [parser, DFG_python]
     else:
         parser = []
@@ -1367,7 +1374,7 @@ def main(input_path, output_dir, ignore_dir_pattern, ignore_file_pattern, requir
                     dir_info["software_type"] = "not found"
         if license_detection:
             try:
-                licenses_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "licenses")
+                licenses_path = str(Path(__file__).parent / "licenses")
                 license_text = extract_license(input_path)
                 rank_list = detect_license(license_text, licenses_path)
                 dir_info["license"] = {}

inspect4py/resources/my-languages.so

@@ -55,7 +55,8 @@ def extract_directory_tree(input_path, ignore_dirs, ignore_files, visual=0):
     ignore_set = tuple(list(ignore_dirs) + list(ignore_files) + ignore_set)
     if visual:
         paths = DisplayablePath.make_tree(Path(input_path), criteria=lambda
-            path: True if path.name not in ignore_set and not os.path.join("../", path.name).endswith(".pyc") else False)
+            path: True if path.name not in ignore_set and not os.path.join("../", path.name).endswith(
+            ".pyc") else False)
         for path in paths:
             print(path.displayable())
     return get_directory_structure(input_path, ignore_set)
@@ -75,7 +76,7 @@ def prune_json(json_dict):
     else:
         for a, b in json_dict.items():
             if a == "ast" and b:
-                final_dict[a] = b # Avoid pruning AST fields
+                final_dict[a] = b  # Avoid pruning AST fields
                 continue
             if b or isinstance(b, bool):
                 if isinstance(b, dict):
@@ -100,14 +101,13 @@ def extract_requirements(input_path):
         # Answering yes (echo y), we allow searching for PyPI
         # for the missing modules and filter some unnecessary modules.
 
-
-        #print(sys.version_info)   
-        if sys.version_info[0] <=3 and sys.version_info[1]<=9:
+        # print(sys.version_info)
+        if sys.version_info[0] <= 3 and sys.version_info[1] <= 9:
             cmd = 'echo y | pigar -P ' + input_path + ' -p ' + file_name
         else:
             cmd = ' pigar generate ' + input_path + ' -f ' + file_name + ' --question-answer yes --auto-select'
-       
-        #print("-----> cmd: %s" %cmd)
+
+        # print("-----> cmd: %s" %cmd)
         proc = subprocess.Popen(cmd.encode('utf-8'), shell=True, stdin=subprocess.PIPE,
                                 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         stdout, stderr = proc.communicate()
@@ -118,7 +118,7 @@ def extract_requirements(input_path):
         for line in lines:
             try:
                 if line != "\n":
-                    if " == " in line: 
+                    if " == " in line:
                         splitLine = line.split(" == ")
                     else:
                         splitLine = line.split("==")
@@ -128,8 +128,8 @@ def extract_requirements(input_path):
 
         # Note: Pigar requirement file is being deleted
         # in the future we might want to keep it (just commenting the line bellow)
-        #os.system('rm ' + file_name)
-        #print("Exracted requirements :%s" %req_dict)
+        # os.system('rm ' + file_name)
+        # print("Exracted requirements :%s" %req_dict)
         return req_dict
 
     except:
@@ -175,7 +175,7 @@ def extract_software_invocation(dir_info, dir_tree_info, input_path, call_list,
     body_only_files = []
     flag_service_main = 0
     for key in dir_info:  # filter (lambda key: key not in "directory_tree", dir_info):
-        if key!="requirements" and key!="directory_tree": # Note: We need to filter out directory_tree
+        if key != "requirements" and key != "directory_tree":  # Note: We need to filter out directory_tree
             for elem in dir_info[key]:
                 if elem["main_info"]["main_flag"]:
                     flag_service_main = 0
@@ -188,7 +188,7 @@ def extract_software_invocation(dir_info, dir_tree_info, input_path, call_list,
                         try:
                             # 2. Exploration for services in files with "mains"
                             flag_service, software_invocation_info = service_check(elem, software_invocation_info,
-                                                                               server_dependencies, "main", readme)
+                                                                                   server_dependencies, "main", readme)
                         except:
                             main_files.append(elem["file"]["path"])
 
@@ -209,19 +209,19 @@ def extract_software_invocation(dir_info, dir_tree_info, input_path, call_list,
 
     # this list (of lists) stores the mains that each main import
     import_mains = []
-   
+
     # this list (of lists) stores the mains that each main is imported by
-    imported_by = [None]*len(main_files)
+    imported_by = [None] * len(main_files)
 
     # 3. Exploration for main scripts
     for m in range(0, len(main_files)):
         m_calls = find_file_calls(main_files[m], call_list)
         # HERE I STORE WHICH OTHER MAIN FILES CALLS EACH "M" MAIN_FILE
         m_imports = extract_relations(main_files[m], m_calls, main_files, call_list)
-      
+
         # storing those m_imports in the import_mains[m]
         import_mains.append(m_imports)
-      
+
         for m_i in m_imports:
             m_secondary[main_files.index(m_i)] = 1
 
@@ -286,7 +286,6 @@ def extract_software_invocation(dir_info, dir_tree_info, input_path, call_list,
     return software_invocation_info
 
 
-
 def generate_output_html(pruned_json, output_file_html):
     """
     Method to generate a simple HTML view of the obtained JSON.
@@ -331,9 +330,9 @@ def list_functions_classes_from_module(m, path):
 
         type = "internal"
     except:
-        
-        #module = __import__(m)
-        #functions = dir(module)
+
+        # module = __import__(m)
+        # functions = dir(module)
         type = "external"
     return functions, classes, type
 
@@ -352,22 +351,22 @@ def type_module(m, i, path):
         return "internal"
     else:
         if m:
-           m = m.replace(".", "/")
-           file_module = abs_repo_path + "/" + m + ".py"
-           file_module_path = Path(file_module)
-           if file_module_path.is_file():
-               return "internal"
-           else:
-               file_module = abs_repo_path + "/" + m + "/main.py"
-               file_module_path = Path(file_module)
-               if file_module_path.is_file():
-                   return "internal"
-               else:
-                   return "external"
+            m = m.replace(".", "/")
+            file_module = abs_repo_path + "/" + m + ".py"
+            file_module_path = Path(file_module)
+            if file_module_path.is_file():
+                return "internal"
+            else:
+                file_module = abs_repo_path + "/" + m + "/main.py"
+                file_module_path = Path(file_module)
+                if file_module_path.is_file():
+                    return "internal"
+                else:
+                    return "external"
         else:
             dir_module = abs_repo_path + "/" + i
             if os.path.exists(dir_module):
-                   return "internal"
+                return "internal"
             else:
                 return "external"
 
@@ -419,7 +418,8 @@ def call_list_dir(dir_info):
             call_list[dir][file_path]["body"] = extract_call_functions(file_info, body=1)
             call_list[dir][file_path]["classes"] = {}
             for class_n in file_info["classes"]:
-                call_list[dir][file_path]["classes"][class_n] = extract_call_methods(file_info["classes"][class_n]["methods"])
+                call_list[dir][file_path]["classes"][class_n] = extract_call_methods(
+                    file_info["classes"][class_n]["methods"])
     return call_list
 
 
@@ -433,8 +433,8 @@ def find_file_calls(file_name, call_list):
 def find_module_calls(module, call_list):
     for dir in call_list:
         for elem in call_list[dir]:
-            if "/"+module+"." in elem:
-                #print("---MODULE %s, elem %s, giving call_list[%s][%s]" %(module, elem, dir, elem))
+            if "/" + module + "." in elem:
+                # print("---MODULE %s, elem %s, giving call_list[%s][%s]" %(module, elem, dir, elem))
                 return call_list[dir][elem]
 
             # DFS algorithm - Allowing up to 2 levels of depth.
@@ -457,7 +457,7 @@ def file_in_call(base, call, file, m_imports, call_list, orig_base, level):
     elif orig_base in call:
         return 0
 
-    elif level < level_depth and call!="":
+    elif level < level_depth and call != "":
         m_calls_extern = {}
         module_base = call.split(".")[0]
         module_base = module_base + "."
@@ -522,7 +522,7 @@ def extract_relations(file_name, m_calls, main_files, call_list):
                 level = 0
                 flag_found = extract_data(base, m_calls[m_c], file, m_imports, flag_found, call_list, orig_base, level)
                 if flag_found:
-                    #return m_imports
+                    # return m_imports
                     break
 
     return m_imports
@@ -622,6 +622,7 @@ def rank_software_invocation(soft_invocation_info_list):
         entry["ranking"] = position
     return soft_invocation_info_list
 
+
 def ast_to_json(ast_obj):
     """
     Function to convert the AST object into JSON format.
@@ -631,6 +632,7 @@ def ast_to_json(ast_obj):
     ast_generator.tree = ast_obj
     return ast_generator.generate_ast()
 
+
 def ast_to_source_code(ast_obj):
     """
     Function to convert the AST object into source code.
@@ -650,8 +652,8 @@ def dice_coefficient(a, b):
     if len(b) == 1:
         b = b + u"."
 
-    a_bigrams = {a[i : i + 2] for i in range(len(a) - 1)}
-    b_bigrams = {b[i : i + 2] for i in range(len(b) - 1)}
+    a_bigrams = {a[i: i + 2] for i in range(len(a) - 1)}
+    b_bigrams = {b[i: i + 2] for i in range(len(b) - 1)}
 
     overlap = len(a_bigrams & b_bigrams)
     dice_coeff = overlap * 2.0 / (len(a_bigrams) + len(b_bigrams))
@@ -727,6 +729,7 @@ def detect_license(license_text, licenses_path, threshold=0.9):
 
     return sorted(rank_list, key=lambda t: t[1], reverse=True)
 
+
 def extract_readme(input_path: str, output_dir: str) -> dict:
     """
     Function to extract content of all readme file under the input directory.
@@ -744,6 +747,7 @@ def extract_readme(input_path: str, output_dir: str) -> dict:
 
     return readme_files
 
+
 def get_github_metadata(input_path: str) -> dict:
     """
     Function to extract metadata from the remote repository using Github api.
@@ -773,8 +777,9 @@ def get_github_metadata(input_path: str) -> dict:
 
     return github_metadata
 
+
 def find_index_init(depInfo, calls, class_init):
-    index_remove=[]
+    index_remove = []
     for dep in depInfo:
         if dep["type_element"] == "class":
             if dep["import"] in calls:
@@ -786,19 +791,21 @@ def find_index_init(depInfo, calls, class_init):
             index_remove.append(calls.index(i))
     return index_remove
 
+
 def update_list_calls(info, index_remove):
-    updated_calls=[]
+    updated_calls = []
     for i in range(0, len(info["calls"])):
         if i in index_remove:
             continue
         updated_calls.append(info["calls"][i])
     ### These lines are for removing duplicate calls
     res = []
-    for i in updated_calls :
+    for i in updated_calls:
         if i not in res:
             res.append(i)
     return res
 
+
 def tree_to_variable_index(root_node, index_to_code):
     if (len(root_node.children) == 0 or root_node.type == 'string') and root_node.type != 'comment':
         index = (root_node.start_point, root_node.end_point)
@@ -813,6 +820,7 @@ def tree_to_variable_index(root_node, index_to_code):
             code_tokens += tree_to_variable_index(child, index_to_code)
         return code_tokens
 
+
 def DFG_python(root_node, index_to_code, states):
     assignment = ['assignment', 'augmented_assignment', 'for_in_clause']
     if_statement = ['if_statement']
@@ -979,7 +987,8 @@ def DFG_python(root_node, index_to_code, states):
                 temp, states = DFG_python(child, index_to_code, states)
                 DFG += temp
 
-        return sorted(DFG,key=lambda x:x[1]),states
+        return sorted(DFG, key=lambda x: x[1]), states
+
 
 def tree_to_variable_index(root_node, index_to_code):
     if (len(root_node.children) == 0 or root_node.type == 'string') and root_node.type != 'comment':
@@ -1009,6 +1018,7 @@ def index_to_code_token(index, code):
         s += code[end_point[0]][:end_point[1]]
     return s
 
+
 def tree_to_token_index(root_node):
     if (len(root_node.children) == 0 or root_node.type == 'string') and root_node.type != 'comment':
         return [(root_node.start_point, root_node.end_point)]
@@ -1018,35 +1028,36 @@ def tree_to_token_index(root_node):
             code_tokens += tree_to_token_index(child)
         return code_tokens
 
-def extract_dataflow(code, parser,lang):
-    #obtain dataflow
-    if lang=="php":
-        code="<?php"+code+"?>"
+
+def extract_dataflow(code, parser, lang):
+    # obtain dataflow
+    if lang == "php":
+        code = "<?php" + code + "?>"
     try:
-        tree = parser[0].parse(bytes(code,'utf8'))
+        tree = parser[0].parse(bytes(code, 'utf8'))
         root_node = tree.root_node
-        tokens_index=tree_to_token_index(root_node)
-        code=code.split('\n')
-        code_tokens=[index_to_code_token(x,code) for x in tokens_index]
-        index_to_code={}
-        for idx,(index,code) in enumerate(zip(tokens_index,code_tokens)):
-            index_to_code[index]=(idx,code)
+        tokens_index = tree_to_token_index(root_node)
+        code = code.split('\n')
+        code_tokens = [index_to_code_token(x, code) for x in tokens_index]
+        index_to_code = {}
+        for idx, (index, code) in enumerate(zip(tokens_index, code_tokens)):
+            index_to_code[index] = (idx, code)
         try:
-            DFG,_=parser[1](root_node,index_to_code,{})
+            DFG, _ = parser[1](root_node, index_to_code, {})
         except:
-            DFG=[]
-        DFG=sorted(DFG,key=lambda x:x[1])
-        indexs=set()
+            DFG = []
+        DFG = sorted(DFG, key=lambda x: x[1])
+        indexs = set()
         for d in DFG:
-            if len(d[-1])!=0:
+            if len(d[-1]) != 0:
                 indexs.add(d[1])
             for x in d[-1]:
                 indexs.add(x)
-        new_DFG=[]
+        new_DFG = []
         for d in DFG:
             if d[1] in indexs:
                 new_DFG.append(d)
-        dfg=new_DFG
+        dfg = new_DFG
     except:
-        dfg=[]
-    return code_tokens, dfg
+        dfg = []
+    return code_tokens, dfg

inspect4py/utils.py

@@ -9,5 +9,6 @@ requires = [
     "setuptools==54.2.0",
     "json2html",
     "configparser",
+    "tree-sitter"
 ]
 build-backend = "setuptools.build_meta"