From 771b515aebc36e0abc6393036951d2ba80bcca6d Mon Sep 17 00:00:00 2001 From: Lance Xu Date: Mon, 6 Nov 2023 09:20:05 -0500 Subject: [PATCH] Add rendering functions to report tool This update causes report to automatically generate an index.html file in the report_analysis folder. Point your browser to this index.html file to render the report_analysis to be graphically. Both Correlated Results and Plugin Results have its sub-items. Clicking items will show corresponding content in the content area. Test Plan: PASS: Verify the current report tool functions is not affected PASS: Verify the index.html is generated in the collect bundle folder PASS: Verify System Info can be read correctly PASS: Verify users can add/remove items in left panel of System Info PASS: Verify Correlated Results and Plugin Results are shown PASS: Verify the subitems of the Results are clickable and shown PASS: Verify the menus are expandable/collapsible PASS: Verify vertical scrollbar is applied when overflow occurs PASS: Verify system info for a selectable list of hosts are displayed PASS: Verify handling of long hostnames PASS: Verify handling of long list of hosts PASS: Verify controller-0 is printed when loading PASS: Verify correlated summary is printed when loading PASS: Verify correlated summary is printed when 'Correlated Results' menu title is selected PASS: Verify correlated results file contents are displayed by clicking on the 'Correlated Results' groups PASS: Verify plugin results summary is printed when 'Plugin Results' menu title is selected PASS: Verify selecting additional hosts adds to that list while deselecting them removes PASS: Verify plugin results file contents can be displayed by clicking on the 'Plugin Results group. PASS: Verify page is refreshed when System Information is clicked PASS: Verify index.html is created for all report bundle pointer options ; -b , -d , -f PASS: Verify index.html is created and included in the report analysis when report is run as part of collect PASS: Verify the generated html with css content will bring no error in console Story: 2010533 Task: 49041 Change-Id: Icf9c727b4f9418e03c0c7e273c8477e5ef9480ac Signed-off-by: Lance Xu --- .../debian-scripts/report/execution_engine.py | 4 +- .../collector/debian-scripts/report/render.py | 604 ++++++++++++++++++ .../collector/debian-scripts/report/report.py | 4 + tools/collector/debian/deb_folder/rules | 1 + 4 files changed, 612 insertions(+), 1 deletion(-) create mode 100755 tools/collector/debian-scripts/report/render.py diff --git a/tools/collector/debian-scripts/report/execution_engine.py b/tools/collector/debian-scripts/report/execution_engine.py index d165a01d..654b1c8d 100755 --- a/tools/collector/debian-scripts/report/execution_engine.py +++ b/tools/collector/debian-scripts/report/execution_engine.py @@ -267,7 +267,9 @@ class ExecutionEngine: system_info_output, self.hosts, True) - for host_dir in self.host_dirs: + start_index = self.active_controller_directory is None + + for host_dir in self.host_dirs[start_index:]: if host_dir != self.active_controller_directory: hostname = re.sub(regex_chop_bundle_date, "", os.path.basename(host_dir)) diff --git a/tools/collector/debian-scripts/report/render.py b/tools/collector/debian-scripts/report/render.py new file mode 100755 index 00000000..72aae54b --- /dev/null +++ b/tools/collector/debian-scripts/report/render.py @@ -0,0 +1,604 @@ +######################################################################## +# +# Copyright (c) 2023 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +######################################################################## +# +# This file contains the Render function +# The Rendering tool visualizes the collect bundle and generates index.html file +# +######################################################################## + +from datetime import datetime +import os + + +def extract_section(log_contents, start_phrase): + """extract the correlated or plugin content of the summary + + Parameters: + log_contents (string): content of the log + start_phrase (string): the name of the section extracted + """ + start = log_contents.find(start_phrase) + if start == -1: + return "" + end = log_contents.find("\n\n", start) + if end == -1: + end = len(log_contents) + return log_contents[start:end].strip() + + +def remove_timestamp(text): + """remove timestamp of summary message + + Parameters: + text (string): the summary message + """ + lines = text.split('\n') + temp = [] + for line in lines: + split_string = line.split(' ', 1) + # check if the first part is time format, then remove if it is + if split_string[0] and datetime.fromisoformat(split_string[0]): + temp.append(split_string[1]) + else: + temp.append(line) + final_text = '\n'.join(temp) + return final_text + + +def remove_emptyinfo(text): + """ remove 'INFO' text of summary message + + Parameters: + text (string): the summary message + """ + lines = text.split('\n') + temp = [] + for line in lines: + if line.strip() != 'INFO:': + temp.append(line) + final_text = '\n'.join(temp) + return final_text + + +def process_section(section, title): + """return text with timestamp and INFO: removed + + Parameters: + section (string): the message of the correlated/plugins section + title (string): correlated/plugin results + """ + section = section[len(title):] + section = remove_timestamp(section) + section = remove_emptyinfo(section) + return section + + +def classify_node(data): + """classify node type in system_info summary + + Parameters: + data (string): the summary of system_info + """ + node_type = '' + for item in data: + if 'Node Type' in item: + node_type = item.split(':')[-1].strip().lower() + return node_type + + +def controller_sort(x): + """sort the controller, place the controller-0 first + + Parameters: + x (list): list of controller info + """ + return x[0] != 'controller-0' + + +def html_css(): + """static css code of the rendering tool + + iframe, textarea: the content panel showing information + #show-worker: the show more worker button + .container-menu: the overall layout of the page + .menu: the sidebar menu of the page + #correlated-results-toggle, #plugin-results-toggle: +/- button for results menu + """ + html_content_css = """ + + + + Report Analysis + + + """ + return html_content_css + + +def html_script(): + """static script code + + Functions: + toggleContent: show content in System Info section + toggleSub: show/hide submenus in correlated/plugin results + toggleMenu: show the correlated/plugin summary + showContentStorage: display content of selected storage item + showContentWorker: display content of selected worker item + showContentTwo: display content of result section + """ + html_content_script = """ + + + """ + return html_content_script + + +def html_info(sys_section): + """system info part generation + reads from plugin/system_info and show by different types + order: controller, storage(if there exists), worker(if there exists) + + Parameters: + sys_section (string): the summary of system_info + """ + controller_section = [] + storage_section = [] + worker_section = [] + + for i in sys_section: + section_lines = i.strip().split("\n") + section_type = classify_node(section_lines) + + if "controller" == section_type: + controller_section.append(section_lines) + + if "storage" == section_type: + storage_section.append(section_lines) + + if "worker" == section_type: + worker_section.append(section_lines) + + controller_section = sorted(controller_section, key=controller_sort) + + controller_zero = controller_section.pop(0) + + sections = { + "controller": controller_section, + "storage": storage_section, + "worker": worker_section + } + + html_content_one = "" + + html_content_one += """ + +
+
+
    + System Information + """ + + html_content_one += "
  • " + html_content_one += """
    """ + html_content_one += """
    -
    controller-0
    """ + for i in range(len(controller_section)): + controlname = controller_section[i][0] + html_content_one += f'
    ' + html_content_one += f'
    +
    {controlname}
    ' + html_content_one += "
    • " + + if storage_section: + html_content_one += """
  • """ + html_content_one += """
    -
    Storage
      """ + for i in range(len(storage_section)): + storagename = storage_section[i][0] + html_content_one += f'
      +
      {storagename}
      ' + html_content_one += "
    • " + + if worker_section: + html_content_one += """
  • """ + html_content_one += """
    -
    Workers
      """ + + max_workers_to_display = min(len(worker_section), 5) + for i in range(len(worker_section)): + workername = worker_section[i][0] + if i < max_workers_to_display: + html_content_one += f'
      +
      {workername}
      ' + else: + html_content_one += f'' + + if len(worker_section) > 5: + html_content_one += """""" + + html_content_one += "
    • " + + html_content_one += """
""" + + # controller-0 + html_content_one += """
""" + for i in controller_zero: + html_content_one += f'{i}' + html_content_one += "
" + html_content_one += "
" + + for section_type, section_list in sections.items(): + for i, section in enumerate(section_list): + if section_type == "controller": + div_id = f"{section_type}-{i + 1}" + else: + div_id = f"{section_type}-{i}" + html_content_one += f'" + + html_content_one += "

""" + return html_content_one + + +def html_result(log_contents, output_dir): + """result part generation in the menu-content style + generates correlated results, plugin results, and the items under them + subitems for plugins and correlated results under separate menus + + Parameters: + log_contents (string): content of the summary + output_dir (string): the location of output + """ + # Extract sections from the log + plugin_section = extract_section(log_contents, 'Plugin Results:') + correlated_section = extract_section(log_contents, 'Correlated Results:') + + # Process the extracted sections + plugin_section = process_section(plugin_section, 'Plugin Results:') + correlated_section = process_section(correlated_section, 'Correlated Results:') + + # HTML part + correlated_directory = os.path.join(os.getcwd(), output_dir) + os.chdir(correlated_directory) + correlated_items = [] + for file in os.listdir(correlated_directory): + if os.path.isfile(file) and '.' not in file: + correlated_items.append({'name': file, 'id': f'content-item-{file}'}) + + plugin_directory = os.path.join(correlated_directory, 'plugins') + os.chdir(plugin_directory) + + plugin_items = [] + for file in os.listdir(plugin_directory): + if os.path.isfile(file) and file != "system_info": + plugin_items.append({'name': file, 'id': f'content-item-{file}'}) + + html_content_two = "" + + html_content_two += """ +
+
+
""" + html_content_two += """
""" + + for item in correlated_items: + html_content_two += f'

{item["name"].capitalize()}

' + + for item in plugin_items: + html_content_two += f'

{item["name"].capitalize()}

' + + html_content_two += f'

Correlated Results

' + html_content_two += f'

Plugin Results

' + html_content_two += """ +
+
+ + """ + + return html_content_two + + +# main +def main(input_dir, output_dir): + reportlog_path = os.path.join(output_dir, 'report.log') + with open(reportlog_path, 'r') as file: + log_contents = file.read() + + sysinfo_path = os.path.join(output_dir, 'plugins/system_info') + with open(sysinfo_path, 'r') as file: + sysinfo_contents = file.read() + + # pre-set html file path + html_file = os.path.abspath(os.path.join(output_dir, 'index.html')) + + sys_section = sysinfo_contents.strip().split("\n\n") + html_content = html_css() + html_info(sys_section) + html_result(log_contents, output_dir) + html_script() + html_content = html_content.format() + + # write the HTML content to file + with open(html_file, "w") as file: + file.write(html_content) diff --git a/tools/collector/debian-scripts/report/report.py b/tools/collector/debian-scripts/report/report.py index fc1e52ad..4ab4044e 100755 --- a/tools/collector/debian-scripts/report/report.py +++ b/tools/collector/debian-scripts/report/report.py @@ -121,6 +121,7 @@ import time import algorithms from execution_engine import ExecutionEngine from plugin import Plugin +import render # Globals now = datetime.now(timezone.utc) @@ -963,4 +964,7 @@ else: # analyze the collect bundle engine.execute(obj.plugins, output_dir) +# generate report tool rendering html file +render.main(input_dir, output_dir) + sys.exit() diff --git a/tools/collector/debian/deb_folder/rules b/tools/collector/debian/deb_folder/rules index 7a31076f..318da0c9 100755 --- a/tools/collector/debian/deb_folder/rules +++ b/tools/collector/debian/deb_folder/rules @@ -38,6 +38,7 @@ override_dh_auto_install: install -m 755 -p report/algorithms.py $(ROOT)/usr/local/bin/report/algorithms.py install -m 755 -p report/plugin.py $(ROOT)/usr/local/bin/report/plugin.py install -m 755 -p report/correlator.py $(ROOT)/usr/local/bin/report/correlator.py + install -m 755 -p report/render.py $(ROOT)/usr/local/bin/report/render.py install -m 644 -p report/README $(ROOT)/usr/local/bin/report/README # Report Tool Plugin Algorithms