starlingx/virtual-deployment
Code Issues Proposed changes

Adding pylint to /virtualbox/pybox

Browse Source 
Enabling automatic pylint with tox and zull for each new patchset.

Test plan:
PASS: Run "tox -e pylint" in the terminal, this will:
  - Run pylint in all python files
  - Show the report

Story: 2005051
Task: 47900
Change-Id: I2f66a5f72e3f8746c00aae96287ad3e4edb88e28
Signed-off-by: Lindley Werner <lindley.vieira@encora.com>
This commit is contained in:
Lindley Werner 2023-05-04 17:21:54 -03:00
parent f73ed26012
commit c93f1aa754
19 changed files with 1846 additions and 800 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.zuul.yaml
View File

@ -5,6 +5,8 @@
check: check:
jobs: jobs:
- openstack-tox-linters - openstack-tox-linters
- openstack-tox-pylint
gate: gate:
jobs: jobs:
- openstack-tox-linters - openstack-tox-linters
- openstack-tox-pylint

2
pylint.rc
View File

@ -234,4 +234,4 @@ valid-classmethod-first-arg=cls
[EXCEPTIONS] [EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to # Exceptions that will emit a warning when being caught. Defaults to
# "Exception" # "Exception"
overgeneral-exceptions=Exception overgeneral-exceptions=builtins.Exception

2
requirements/test-requirements.txt
View File

@ -1,4 +1,4 @@
yamllint === 1.32.0 yamllint === 1.32.0
bashate === 2.1.1 bashate === 2.1.1
pylint === 2.13.9 pylint === 2.13.9
tox === 4.6.3

12
tox.ini
View File

@ -4,9 +4,7 @@ minversion = 2.3
skipsdist = True skipsdist = True
[testenv] [testenv]
deps = deps = -r{toxinidir}/requirements/test-requirements.txt
-r{toxinidir}/requirements/test-requirements.txt
-r{toxinidir}/virtualbox/pybox/requirements.txt
allowlist_externals = reno allowlist_externals = reno
[testenv:linters] [testenv:linters]
@ -23,7 +21,6 @@ commands =
-print0 | xargs -0 yamllint" -print0 | xargs -0 yamllint"
bash -c "find {toxinidir} \ bash -c "find {toxinidir} \
-not \( -type d -name .?\* -prune \) \ -not \( -type d -name .?\* -prune \) \
-not \( -type d -path {toxinidir}/toCOPY/mock_overlay -prune \) \
-type f \ -type f \
-not -name \*~ \ -not -name \*~ \
-not -name \*.md \ -not -name \*.md \
@ -33,6 +30,13 @@ commands =
[testenv:pylint] [testenv:pylint]
basepython = python3 basepython = python3
sitepackages = False sitepackages = False
setenv =
BASEPATH = {toxinidir}/virtualbox/pybox
PYTHONPATH= {env:BASEPATH}:{env:BASEPATH}/helper:{env:BASEPATH}/consts:{env:BASEPATH}/utils
deps =
-r{env:BASEPATH}/requirements.txt
{[testenv]deps}
allowlist_externals = pylint
commands = commands =
pylint {posargs} --rcfile=./pylint.rc virtualbox/pybox pylint {posargs} --rcfile=./pylint.rc virtualbox/pybox

61
virtualbox/pybox/Parser.py
View File

@ -1,3 +1,4 @@
# pylint: disable=invalid-name
#!/usr/bin/python3 #!/usr/bin/python3
# #
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
@ -11,7 +12,7 @@ Parser to handle command line arguments
import argparse import argparse
import getpass import getpass
# pylint: disable=too-many-statements
def handle_args(): def handle_args():
""" """
Handle arguments supplied to the command line Handle arguments supplied to the command line
@ -19,11 +20,10 @@ def handle_args():
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
""" #**************************************
************************************** #* Setup type & install configuration *
* Setup type & install configuration * #**************************************
**************************************
"""
parser.add_argument("--setup-type", help= parser.add_argument("--setup-type", help=
""" """
Type of setup: Type of setup:
@ -84,11 +84,10 @@ def handle_args():
type=str, type=str,
default=None) default=None)
""" #******************************************
****************************************** #* Config folders and files configuration *
* Config folders and files configuration * #******************************************
******************************************
"""
parser.add_argument("--iso-location", help= parser.add_argument("--iso-location", help=
""" """
Location of ISO including the filename: Location of ISO including the filename:
@ -143,11 +142,11 @@ def handle_args():
Path to the config file to use Path to the config file to use
""", """,
action='append') action='append')
"""
************************************** #**************************************
* Disk number and size configuration * #* Disk number and size configuration *
************************************** #**************************************
"""
parser.add_argument("--controller-disks", help= parser.add_argument("--controller-disks", help=
""" """
Select the number of disks for a controller VM. default is 3 Select the number of disks for a controller VM. default is 3
@ -178,11 +177,11 @@ def handle_args():
Configure size in MiB of worker disks as a comma separated list. Configure size in MiB of worker disks as a comma separated list.
""", """,
type=str) type=str)
"""
************** #**************
* Networking * #* Networking *
************** #**************
"""
parser.add_argument("--vboxnet-name", help= parser.add_argument("--vboxnet-name", help=
""" """
Which host only network to use for setup. Which host only network to use for setup.
@ -277,11 +276,11 @@ def handle_args():
SX setups. SX setups.
""", """,
type=str) type=str)
"""
****************** #******************
* Custom scripts * #* Custom scripts *
****************** #******************
"""
parser.add_argument("--script1", help= parser.add_argument("--script1", help=
""" """
Name of an executable script file plus options. Name of an executable script file plus options.
@ -326,11 +325,11 @@ def handle_args():
""", """,
default=None, default=None,
type=str) type=str)
"""
************************************** #**************************************
* Other * #* Other *
************************************** #**************************************
"""
parser.add_argument("--list-stages", help= parser.add_argument("--list-stages", help=
""" """
List stages that can be used by autoinstaller. List stages that can be used by autoinstaller.

17
virtualbox/pybox/README.md
View File

@ -136,7 +136,7 @@ will be configured and used.
```shell ```shell
git clone https://opendev.org/starlingx/tools.git git clone https://opendev.org/starlingx/tools.git
cd tools/deployment/virtualbox/pybox cd virtual-deployment/virtualbox/pybox
python3 -m venv venv python3 -m venv venv
source ./venv/bin/activate source ./venv/bin/activate
pip install --upgrade pip pip install --upgrade pip
@ -150,7 +150,7 @@ will be configured and used.
-O $HOME/Downloads/stx-8.iso -O $HOME/Downloads/stx-8.iso
``` ```
5. Now you're ready to run the script. From the `/deployment/virtualbox/pybox` 5. Now you're ready to run the script. From the `/virtualbox/pybox`
folder, do: folder, do:
```shell ```shell
@ -168,8 +168,15 @@ folder, do:
--snapshot --snapshot
``` ```
The script takes a while to do all the things (from creating a VM and The script takes a while to do all the things (from creating a VM and
installing an OS in it to configuring StarlingX). Several restarts might installing an OS in it to configuring StarlingX). Several restarts might
occur, and you might see a VirtualBox with a prompt. You don't need to type occur, and you might see a VirtualBox with a prompt. You don't need to type
anything. While the installation script is running it will take care of anything. While the installation script is running it will take care of
everything for you. everything for you.
## Pybox folder structure
.
├── configs/aio-sx: Contains scripts and configs to set up a controller/worker
├── consts: This folder contains modules for managing virtual lab environments, including classes for Lab, Subnets, NICs, OAM, Serial, nodes, and HostTimeout.
├── helper: This folder contains modules for interacting with a StarlingX controller-0 server via a serial connection, configuring system settings, and managing virtual machines using VirtualBox.
└── utils: This folder contains modules for initializing logging, tracking and reporting KPIs, connecting and communicating with remote hosts via local domain socket, and sending files and directories to remote servers using rsync and paramiko libraries.

29
virtualbox/pybox/consts/env.py
View File

@ -3,6 +3,11 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module contains a class named Lab and some supporting code.
The Lab class represents a virtual lab and has a dictionary attribute VBOX
containing information about the virtual machines in the lab.
"""
import getpass import getpass
from sys import platform from sys import platform
@ -10,18 +15,22 @@ import os
user = getpass.getuser() user = getpass.getuser()
if platform == 'win32' or platform == 'win64': if platform in ("win32", "win64"):
LOGPATH = 'C:\\Temp\\pybox_logs' LOGPATH = "C:\\Temp\\pybox_logs"
PORT = 10000 PORT = 10000
else: else:
homedir = os.environ['HOME'] homedir = os.environ["HOME"]
LOGPATH = '{}/vbox_installer_logs'.format(homedir) LOGPATH = f"{homedir}/vbox_installer_logs"
class Lab: #pylint: disable=too-few-public-methods
"""The `Lab` class represents a virtual lab and contains a dictionary attribute
`VBOX` with information about the virtual machines in the lab."""
class Lab:
VBOX = { VBOX = {
'floating_ip': '10.10.10.7', "floating_ip": "10.10.10.7",
'controller-0_ip': '10.10.10.8', "controller-0_ip": "10.10.10.8",
'controller-1_ip': '10.10.10.9', "controller-1_ip": "10.10.10.9",
'username': 'sysadmin', "username": "sysadmin",
'password': 'Li69nux*', "password": "Li69nux*",
} }

190
virtualbox/pybox/consts/networking.py
View File

@ -1,78 +1,194 @@
# pylint: disable=too-few-public-methods
#!/usr/bin/python3 #!/usr/bin/python3
# #
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module defines several classes and dictionaries that contain information related
to virtual machines in a lab environment.
Classes:
- `Subnets`: A class containing dictionaries for IPv4 and IPv6 subnets.
- `NICs`: A class containing dictionaries for NIC configurations of different types of
nodes in the virtual environment, such as `CONTROLLER`, `COMPUTE`, and `STORAGE`.
- `OAM`: A class containing an IP address and netmask for the out-of-band management (OAM) network.
- `Serial`: A class containing configurations for the serial ports.
"""
from sys import platform from sys import platform
class Subnets: class Subnets:
"""The `Subnets` class contains dictionaries for IPv4 and IPv6 subnets for the
management, infrastructure, and OAM networks."""
IPV4 = { IPV4 = {
'mgmt_subnet': '192.168.204.0/24', "mgmt_subnet": "192.168.204.0/24",
'infra_subnet': '192.168.205.0/24', "infra_subnet": "192.168.205.0/24",
'oam_subnet': '10.10.10.0/24' "oam_subnet": "10.10.10.0/24",
} }
IPV6 = { IPV6 = {
'mgmt_subnet': 'aefd::/64', "mgmt_subnet": "aefd::/64",
'infra_subnet': 'aced::/64', "infra_subnet": "aced::/64",
'oam_subnet': 'abcd::/64' "oam_subnet": "abcd::/64",
} }
class NICs: class NICs:
if platform == 'win32' or platform == 'win64': """The `NICs` class contains dictionaries for NIC configurations of different types
of nodes in the virtual environment, such as `CONTROLLER`, `COMPUTE`, and `STORAGE`."""
if platform in ("win32", "win64"):
CONTROLLER = { CONTROLLER = {
'node_type': 'controller', "node_type": "controller",
'1': {'nic': 'hostonly', 'intnet': '', 'nictype': '82540EM', 'nicpromisc': 'deny', 'hostonlyadapter': 'VirtualBox Host-Only Ethernet Adapter'}, "1": {
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nic": "hostonly",
'3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "intnet": "",
'4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nictype": "82540EM",
"nicpromisc": "deny",
"hostonlyadapter": "VirtualBox Host-Only Ethernet Adapter",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-data1",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"4": {
"nic": "intnet",
"intnet": "intnet-data2",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
} }
else: else:
CONTROLLER = { CONTROLLER = {
'node_type': 'controller', "node_type": "controller",
'1': {'nic': 'hostonly', 'intnet': '', 'nictype': '82540EM', 'nicpromisc': 'deny', 'hostonlyadapter': 'vboxnet0'}, "1": {
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nic": "hostonly",
'3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "intnet": "",
'4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nictype": "82540EM",
"nicpromisc": "deny",
"hostonlyadapter": "vboxnet0",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-data1",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"4": {
"nic": "intnet",
"intnet": "intnet-data2",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
} }
COMPUTE = { COMPUTE = {
'node_type': 'compute', "node_type": "compute",
'1': {'nic': 'intnet', 'intnet': 'intnet-unused1', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "1": {
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nic": "intnet",
'3': {'nic': 'intnet', 'intnet': 'intnet-data1', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "intnet": "intnet-unused1",
'4': {'nic': 'intnet', 'intnet': 'intnet-data2', 'nictype': 'virtio', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-data1",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"4": {
"nic": "intnet",
"intnet": "intnet-data2",
"nictype": "virtio",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
} }
STORAGE = { STORAGE = {
'node_type': 'storage', "node_type": "storage",
'1': {'nic': 'intnet', 'intnet': 'intnet-unused', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "1": {
'2': {'nic': 'intnet', 'intnet': 'intnet-management', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "nic": "intnet",
'3': {'nic': 'intnet', 'intnet': 'intnet-infra', 'nictype': '82540EM', 'nicpromisc': 'allow-all', 'hostonlyadapter': 'none'}, "intnet": "intnet-unused",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"2": {
"nic": "intnet",
"intnet": "intnet-management",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
"3": {
"nic": "intnet",
"intnet": "intnet-infra",
"nictype": "82540EM",
"nicpromisc": "allow-all",
"hostonlyadapter": "none",
},
} }
class OAM: class OAM:
"""The `OAM` class contains an IP address and netmask for the out-of-band
management (OAM) network."""
OAM = { OAM = {
'ip': '10.10.10.254', "ip": "10.10.10.254",
'netmask': '255.255.255.0', "netmask": "255.255.255.0",
} }
class Serial: class Serial:
if platform == 'win32' or platform == 'win64': """The `Serial` class contains configurations for the serial ports."""
if platform in ("win32", "win64"):
SERIAL = { SERIAL = {
'uartbase': '0x3F8', "uartbase": "0x3F8",
'uartport': '4', "uartport": "4",
'uartmode': 'tcpserver', "uartmode": "tcpserver",
'uartpath': '10000' "uartpath": "10000",
} }
else: else:
SERIAL = { SERIAL = {
'uartbase': '0x3F8', "uartbase": "0x3F8",
'uartport': '4', "uartport": "4",
'uartmode': 'server', "uartmode": "server",
'uartpath': '/tmp/' "uartpath": "/tmp/",
} }

9
virtualbox/pybox/consts/node.py
View File

@ -3,8 +3,15 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module contains dictionaries for different types of nodes in a virtual environment,
such as CONTROLLER_CEPH, CONTROLLER_LVM, CONTROLLER_AIO, COMPUTE, and STORAGE.
"""
class Nodes: #pylint: disable=too-few-public-methods
"""The `Nodes` class contains dictionaries for different types of nodes in a
virtual environment."""
class Nodes:
CONTROLLER_CEPH = { CONTROLLER_CEPH = {
'node_type': 'controller-STORAGE', 'node_type': 'controller-STORAGE',
'memory': 12288, 'memory': 12288,

9
virtualbox/pybox/consts/timeout.py
View File

@ -3,8 +3,15 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module contains the HostTimeout class, which provides timeout values (in seconds)
for various operations on a host.
"""
class HostTimeout: #pylint: disable=too-few-public-methods
"""The `HostTimeout` class provides timeout values (in seconds) for various
operations on a host."""
class HostTimeout:
CONTROLLER_UNLOCK = 3600+1800 CONTROLLER_UNLOCK = 3600+1800
REBOOT = 900 REBOOT = 900
INSTALL = 3600 INSTALL = 3600

46
virtualbox/pybox/helper/host_helper.py
View File

@ -3,6 +3,12 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module provides functions to interact with a StarlingX controller-0 server via a
serial connection. The functions can be used to perform operations such as unlocking,
locking, rebooting, and installing a host. The module uses streamexpect library to
facilitate stream parsing.
"""
import time import time
import streamexpect import streamexpect
@ -21,15 +27,17 @@ def unlock_host(stream, hostname):
- Check that host is locked - Check that host is locked
- Unlock host - Unlock host
""" """
LOG.info("#### Unlock %s", hostname) LOG.info("#### Unlock %s", hostname)
serial.send_bytes(stream, "system host-list | grep {}".format(hostname), expect_prompt=False) serial.send_bytes(stream, f"system host-list | grep {hostname}", expect_prompt=False)
try: try:
serial.expect_bytes(stream, "locked") serial.expect_bytes(stream, "locked")
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
LOG.info("Host %s not locked", hostname) LOG.info("Host %s not locked", hostname)
return 1 return 1
serial.send_bytes(stream, "system host-unlock {}".format(hostname), expect_prompt=False) serial.send_bytes(stream, f"system host-unlock {hostname}", expect_prompt=False)
LOG.info("Unlocking %s", hostname) LOG.info("Unlocking %s", hostname)
return None
def lock_host(stream, hostname): def lock_host(stream, hostname):
@ -42,15 +50,17 @@ def lock_host(stream, hostname):
- Check that host is unlocked - Check that host is unlocked
- Lock host - Lock host
""" """
LOG.info("Lock %s", hostname) LOG.info("Lock %s", hostname)
serial.send_bytes(stream, "system host-list |grep {}".format(hostname), expect_prompt=False) serial.send_bytes(stream, f"system host-list |grep {hostname}", expect_prompt=False)
try: try:
serial.expect_bytes(stream, "unlocked") serial.expect_bytes(stream, "unlocked")
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
LOG.info("Host %s not unlocked", hostname) LOG.info("Host %s not unlocked", hostname)
return 1 return 1
serial.send_bytes(stream, "system host-lock {}".format(hostname), expect_prompt="keystone") serial.send_bytes(stream, f"system host-lock {hostname}", expect_prompt="keystone")
LOG.info("Locking %s", hostname) LOG.info("Locking %s", hostname)
return None
def reboot_host(stream, hostname): def reboot_host(stream, hostname):
@ -60,8 +70,9 @@ def reboot_host(stream, hostname):
stream(): stream():
hostname(str): Host to reboot hostname(str): Host to reboot
""" """
LOG.info("Rebooting %s", hostname) LOG.info("Rebooting %s", hostname)
serial.send_bytes(stream, "system host-reboot {}".format(hostname), expect_prompt=False) serial.send_bytes(stream, f"system host-reboot {hostname}", expect_prompt=False)
serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT) serial.expect_bytes(stream, "rebooting", HostTimeout.REBOOT)
@ -77,18 +88,17 @@ def install_host(stream, hostname, host_type, host_id):
time.sleep(10) time.sleep(10)
LOG.info("Installing %s with id %s", hostname, host_id) LOG.info("Installing %s with id %s", hostname, host_id)
if host_type is 'controller': if host_type == 'controller':
serial.send_bytes(stream, serial.send_bytes(stream,
"system host-update {} personality=controller".format(host_id), f"system host-update {host_id} personality=controller",
expect_prompt=False) expect_prompt=False)
elif host_type is 'storage': elif host_type == 'storage':
serial.send_bytes(stream, serial.send_bytes(stream,
"system host-update {} personality=storage".format(host_id), f"system host-update {host_id} personality=storage",
expect_prompt=False) expect_prompt=False)
else: else:
serial.send_bytes(stream, serial.send_bytes(stream,
"system host-update {} personality=compute hostname={}".format(host_id, f"system host-update {host_id} personality=compute hostname={hostname}",
hostname),
expect_prompt=False) expect_prompt=False)
time.sleep(30) time.sleep(30)
@ -99,6 +109,7 @@ def disable_logout(stream):
Args: Args:
stream(stream): stream to cont0 stream(stream): stream to cont0
""" """
LOG.info('Disabling automatic logout') LOG.info('Disabling automatic logout')
serial.send_bytes(stream, "export TMOUT=0") serial.send_bytes(stream, "export TMOUT=0")
@ -108,8 +119,8 @@ def change_password(stream, username="sysadmin", password="Li69nux*"):
changes the default password on initial login. changes the default password on initial login.
Args: Args:
stream(stream): stream to cont0 stream(stream): stream to cont0
""" """
LOG.info('Changing password to Li69nux*') LOG.info('Changing password to Li69nux*')
serial.send_bytes(stream, username, expect_prompt=False) serial.send_bytes(stream, username, expect_prompt=False)
serial.expect_bytes(stream, "Password:") serial.expect_bytes(stream, "Password:")
@ -131,8 +142,8 @@ def login(stream, timeout=600, username="sysadmin", password="Li69nux*"):
""" """
serial.send_bytes(stream, "\n", expect_prompt=False) serial.send_bytes(stream, "\n", expect_prompt=False)
rc = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout) login_result = serial.expect_bytes(stream, "ogin:", fail_ok=True, timeout=timeout)
if rc != 0: if login_result != 0:
serial.send_bytes(stream, "\n", expect_prompt=False) serial.send_bytes(stream, "\n", expect_prompt=False)
if serial.expect_bytes(stream, "~$", timeout=10, fail_ok=True) == -1: if serial.expect_bytes(stream, "~$", timeout=10, fail_ok=True) == -1:
serial.send_bytes(stream, '\n', expect_prompt=False) serial.send_bytes(stream, '\n', expect_prompt=False)
@ -150,11 +161,18 @@ def logout(stream):
Args: Args:
stream(stream): stream to cont0 stream(stream): stream to cont0
""" """
serial.send_bytes(stream, "exit", expect_prompt=False) serial.send_bytes(stream, "exit", expect_prompt=False)
time.sleep(5) time.sleep(5)
def check_password(stream, password="Li69nux*"): def check_password(stream, password="Li69nux*"):
"""
Checks the password.
Args:
stream(stream): Stream to cont0
password(str): password to check.
"""
ret = serial.expect_bytes(stream, 'assword', fail_ok=True, timeout=5) ret = serial.expect_bytes(stream, 'assword', fail_ok=True, timeout=5)
if ret == 0: if ret == 0:
serial.send_bytes(stream, password, expect_prompt=False) serial.send_bytes(stream, password, expect_prompt=False)

42
virtualbox/pybox/helper/install_lab.py
View File

@ -8,11 +8,10 @@ Contains helper functions that will configure basic system settings.
""" """
from consts.timeout import HostTimeout from consts.timeout import HostTimeout
from helper import host_helper
from utils import serial from utils import serial
from utils.install_log import LOG from utils.install_log import LOG
from helper import host_helper
def update_platform_cpus(stream, hostname, cpu_num=5): def update_platform_cpus(stream, hostname, cpu_num=5):
""" """
@ -20,9 +19,14 @@ def update_platform_cpus(stream, hostname, cpu_num=5):
""" """
LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname) LOG.info("Allocating %s CPUs for use by the %s platform.", cpu_num, hostname)
serial.send_bytes(stream, "\nsource /etc/platform/openrc; system host-cpu-modify " serial.send_bytes(
"{} -f platform -p0 {}".format(hostname, cpu_num, stream,
prompt='keystone', timeout=300)) "\nsource /etc/platform/openrc; system host-cpu-modify "
f"{hostname} -f platform -p0 {cpu_num}",
prompt="keystone",
timeout=300,
)
def set_dns(stream, dns_ip): def set_dns(stream, dns_ip):
""" """
@ -30,25 +34,31 @@ def set_dns(stream, dns_ip):
""" """
LOG.info("Configuring DNS to %s.", dns_ip) LOG.info("Configuring DNS to %s.", dns_ip)
serial.send_bytes(stream, "source /etc/platform/openrc; system dns-modify " serial.send_bytes(
"nameservers={}".format(dns_ip), prompt='keystone') stream,
"source /etc/platform/openrc; system dns-modify "
f"nameservers={dns_ip}",
prompt="keystone",
)
def config_controller(stream, config_file=None, password='Li69nux*'): def config_controller(stream, config_file=None, password="Li69nux*"):
""" """
Configure controller-0 using optional arguments Configure controller-0 using optional arguments
""" """
args = '' args = ""
if config_file: if config_file:
args += '--config-file ' + config_file + ' ' args += "--config-file " + config_file + " "
# serial.send_bytes(stream, f'sudo config_controller {args}', expect_prompt=False) # serial.send_bytes(stream, f'sudo config_controller {args}', expect_prompt=False)
serial.send_bytes(stream, 'ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml', expect_prompt=False) serial.send_bytes(
stream,
"ansible-playbook /usr/share/ansible/stx-ansible/playbooks/bootstrap.yml",
expect_prompt=False,
)
host_helper.check_password(stream, password=password) host_helper.check_password(stream, password=password)
ret = serial.expect_bytes(stream, "~$", ret = serial.expect_bytes(stream, "~$", timeout=HostTimeout.LAB_CONFIG)
timeout=HostTimeout.LAB_CONFIG)
if ret != 0: if ret != 0:
LOG.info("Configuration failed. Exiting installer.") LOG.info("Configuration failed. Exiting installer.")
raise Exception("Configcontroller failed") raise Exception("Configcontroller failed") # pylint: disable=E0012, W0719

737
virtualbox/pybox/helper/vboxmanage.py
View File

@ -3,6 +3,9 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module provides functions for managing virtual machines using VirtualBox.
"""
import os import os
import subprocess import subprocess
@ -12,7 +15,6 @@ import time
from sys import platform from sys import platform
from consts import env from consts import env
from utils.install_log import LOG from utils.install_log import LOG
@ -21,59 +23,90 @@ def vboxmanage_version():
Return version of vbox. Return version of vbox.
""" """
version = subprocess.check_output(['vboxmanage', '--version'], stderr=subprocess.STDOUT) version = subprocess.check_output(
["vboxmanage", "--version"], stderr=subprocess.STDOUT
)
return version return version
def vboxmanage_extpack(action="install"): def vboxmanage_extpack():
""" """
This allows you to install, uninstall the vbox extensions" This allows you to install, uninstall the vbox extensions"
""" """
output = vboxmanage_version() output = vboxmanage_version()
version = re.match(b'(.*)r', output) version = re.match(b"(.*)r", output)
version_path = version.group(1).decode('utf-8') version_path = version.group(1).decode("utf-8")
LOG.info("Downloading extension pack") LOG.info("Downloading extension pack")
filename = 'Oracle_VM_VirtualBox_Extension_Pack-{}.vbox-extpack'.format(version_path) filename = f"Oracle_VM_VirtualBox_Extension_Pack-{version_path}.vbox-extpack"
cmd = 'http://download.virtualbox.org/virtualbox/{}/{}'.format(version_path, filename) cmd = f"http://download.virtualbox.org/virtualbox/{version_path}/{filename}"
result = subprocess.check_output(['wget', cmd, '-P', '/tmp'], stderr=subprocess.STDOUT) result = subprocess.check_output(
["wget", cmd, "-P", "/tmp"], stderr=subprocess.STDOUT
)
LOG.info(result) LOG.info(result)
LOG.info("Installing extension pack") LOG.info("Installing extension pack")
result = subprocess.check_output(['vboxmanage', 'extpack', 'install', '/tmp/' + filename, result = subprocess.check_output(
'--replace'], stderr=subprocess.STDOUT) ["vboxmanage", "extpack", "install", "/tmp/" + filename, "--replace"],
stderr=subprocess.STDOUT,
)
LOG.info(result) LOG.info(result)
def get_all_vms(labname, option="vms"): def get_all_vms(labname, option="vms"):
"""
Return a list of virtual machines (VMs) belonging to a specified lab.
Args:
labname (str): The name of the lab to which the VMs belong.
option (str, optional): The VBoxManage command option to use when listing VMs.
Defaults to "vms".
Returns:
list: A list of strings representing the names of the VMs that belong to the specified lab.
"""
initial_node_list = [] initial_node_list = []
vm_list = vboxmanage_list(option) vm_list = vboxmanage_list(option)
labname.encode('utf-8') labname.encode("utf-8")
# Reduce the number of VMs we query # Reduce the number of VMs we query
for item in vm_list: for item in vm_list:
if labname.encode('utf-8') in item and (b'controller-' in item or \ if labname.encode("utf-8") in item and (
b'compute-' in item or b'storage-' in item): b"controller-" in item or b"compute-" in item or b"storage-" in item
initial_node_list.append(item.decode('utf-8')) ):
initial_node_list.append(item.decode("utf-8"))
# Filter by group # Filter by group
node_list = [] node_list = []
group = bytearray('"/{}"'.format(labname), 'utf-8') group = bytearray(f'"/{labname}"', "utf-8")
for item in initial_node_list: for item in initial_node_list:
info = vboxmanage_showinfo(item).splitlines() info = vboxmanage_showinfo(item).splitlines()
for line in info: for line in info:
try: try:
k, v = line.split(b'=') k_value, v_value = line.split(b"=")
except ValueError: except ValueError:
continue continue
if k == b'groups' and v == group: if k_value == b"groups" and v_value == group:
node_list.append(item) node_list.append(item)
return node_list return node_list
def take_snapshot(labname, snapshot_name, socks=None): def take_snapshot(labname, snapshot_name):
"""
Take a snapshot of all VMs belonging to a specified lab.
Args:
labname (str): The name of the lab whose VMs will be snapshotted.
snapshot_name (str): The name of the snapshot to be taken.
Returns:
None
"""
vms = get_all_vms(labname, option="vms") vms = get_all_vms(labname, option="vms")
runningvms = get_all_vms(labname, option="runningvms") runningvms = get_all_vms(labname, option="runningvms")
@ -81,50 +114,103 @@ def take_snapshot(labname, snapshot_name, socks=None):
LOG.info("VMs in lab %s: %s", labname, vms) LOG.info("VMs in lab %s: %s", labname, vms)
LOG.info("VMs running in lab %s: %s", labname, runningvms) LOG.info("VMs running in lab %s: %s", labname, runningvms)
hosts = len(vms) _pause_running_vms(runningvms, vms)
if len(vms) != 0:
vboxmanage_takesnapshot(vms, snapshot_name)
_resume_running_vms(runningvms)
time.sleep(10)
if runningvms:
_wait_for_vms_to_run(labname, runningvms, vms)
def _pause_running_vms(runningvms, vms):
"""Pause running virtual machines.
Args:
runningvms (list): A list of strings representing the names of running virtual machines.
vms (list): A list of strings representing the names of all virtual machines.
Returns:
None
"""
# Pause running VMs to take snapshot
if len(runningvms) > 1: if len(runningvms) > 1:
for node in runningvms: for node in runningvms:
newpid = os.fork() newpid = os.fork()
if newpid == 0: if newpid == 0:
vboxmanage_controlvms([node], "pause") vboxmanage_controlvms([node], "pause")
os._exit(0) os._exit(0) # pylint: disable=protected-access
for node in vms: for node in vms:
os.waitpid(0, 0) os.waitpid(0, 0)
time.sleep(2) time.sleep(2)
if hosts != 0:
vboxmanage_takesnapshot(vms, snapshot_name)
# Resume VMs after snapshot was taken def _resume_running_vms(runningvms):
"""Resume paused virtual machines.
Args:
runningvms (list): A list of strings representing the names of running virtual machines.
Returns:
None
"""
if len(runningvms) > 1: if len(runningvms) > 1:
for node in runningvms: for node in runningvms:
newpid = os.fork() newpid = os.fork()
if newpid == 0: if newpid == 0:
vboxmanage_controlvms([node], "resume") vboxmanage_controlvms([node], "resume")
os._exit(0) os._exit(0) # pylint: disable=protected-access
for node in runningvms: for node in runningvms:
os.waitpid(0, 0) os.waitpid(0, 0)
time.sleep(10) # Wait for VM serial port to stabilize, otherwise it may refuse to connect
if runningvms: def _wait_for_vms_to_run(labname, runningvms, vms):
new_vms = get_all_vms(labname, option="runningvms") """Wait for virtual machines to finish running.
retry = 0
while retry < 20: Args:
LOG.info("Waiting for VMs to come up running after taking snapshot..." labname (str): The name of the lab whose virtual machines are being waited for.
"Up VMs are %s ", new_vms) runningvms (list): A list of strings representing the names of running virtual machines.
if len(runningvms) < len(new_vms): vms (list): A list of strings representing the names of all virtual machines.
time.sleep(1)
new_vms = get_all_vms(labname, option="runningvms") Returns:
retry += 1 None
else: """
LOG.info("All VMs %s are up running after taking snapshot...", vms)
break new_vms = get_all_vms(labname, option="runningvms")
retry = 0
while retry < 20:
LOG.info(
"Waiting for VMs to come up running after taking snapshot..."
"Up VMs are %s ",
new_vms,
)
if len(runningvms) < len(new_vms):
time.sleep(1)
new_vms = get_all_vms(labname, option="runningvms")
retry += 1
else:
LOG.info("All VMs %s are up running after taking snapshot...", vms)
break
def restore_snapshot(node_list, name): def restore_snapshot(node_list, name):
"""
Restore a snapshot of a list of virtual machines.
Args:
node_list (list): A list of strings representing the names of the virtual machines
whose snapshot will be restored.
name (str): The name of the snapshot to restore.
Returns:
None
"""
LOG.info("Restore snapshot of %s for hosts %s", name, node_list) LOG.info("Restore snapshot of %s for hosts %s", name, node_list)
if len(node_list) != 0: if len(node_list) != 0:
vboxmanage_controlvms(node_list, "poweroff") vboxmanage_controlvms(node_list, "poweroff")
@ -147,7 +233,10 @@ def vboxmanage_list(option="vms"):
""" """
This returns a list of vm names. This returns a list of vm names.
""" """
result = subprocess.check_output(['vboxmanage', 'list', option], stderr=subprocess.STDOUT)
result = subprocess.check_output(
["vboxmanage", "list", option], stderr=subprocess.STDOUT
)
vms_list = [] vms_list = []
for item in result.splitlines(): for item in result.splitlines():
vm_name = re.match(b'"(.*?)"', item) vm_name = re.match(b'"(.*?)"', item)
@ -160,10 +249,13 @@ def vboxmanage_showinfo(host):
""" """
This returns info about the host This returns info about the host
""" """
if not isinstance(host, str): if not isinstance(host, str):
host.decode('utf-8') host.decode("utf-8")
result = subprocess.check_output(['vboxmanage', 'showvminfo', host, '--machinereadable'], result = subprocess.check_output(
stderr=subprocess.STDOUT) ["vboxmanage", "showvminfo", host, "--machinereadable"],
stderr=subprocess.STDOUT,
)
return result return result
@ -176,9 +268,21 @@ def vboxmanage_createvm(hostname, labname):
assert labname, "Labname is required" assert labname, "Labname is required"
group = "/" + labname group = "/" + labname
LOG.info("Creating VM %s", hostname) LOG.info("Creating VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'createvm', '--name', hostname, '--register', subprocess.check_output(
'--ostype', 'Linux_64', '--groups', group], [
stderr=subprocess.STDOUT) "vboxmanage",
"createvm",
"--name",
hostname,
"--register",
"--ostype",
"Linux_64",
"--groups",
group,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_deletevms(hosts=None): def vboxmanage_deletevms(hosts=None):
""" """
@ -190,124 +294,295 @@ def vboxmanage_deletevms(hosts=None):
if len(hosts) != 0: if len(hosts) != 0:
for hostname in hosts: for hostname in hosts:
LOG.info("Deleting VM %s", hostname) LOG.info("Deleting VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'unregistervm', hostname, '--delete'], subprocess.check_output(
stderr=subprocess.STDOUT) ["vboxmanage", "unregistervm", hostname, "--delete"],
stderr=subprocess.STDOUT,
)
time.sleep(10) time.sleep(10)
# in case medium is still present after delete # in case medium is still present after delete
vboxmanage_deletemedium(hostname) vboxmanage_deletemedium(hostname)
vms_list = vboxmanage_list("vms") vms_list = vboxmanage_list("vms")
for items in hosts: for items in hosts:
assert items not in vms_list, "The following vms are unexpectedly" \ assert (
"present {}".format(vms_list) items not in vms_list
), f"The following vms are unexpectedly present {vms_list}"
def vboxmanage_hostonlyifcreate(name="vboxnet0", ip=None, netmask=None): def vboxmanage_hostonlyifcreate(name="vboxnet0", oam_ip=None, netmask=None):
""" """
This creates a hostonly network for systems to communicate. This creates a hostonly network for systems to communicate.
""" """
assert name, "Must provide network name" assert name, "Must provide network name"
assert ip, "Must provide an OAM IP" assert oam_ip, "Must provide an OAM IP"
assert netmask, "Must provide an OAM Netmask" assert netmask, "Must provide an OAM Netmask"
LOG.info("Creating Host-only Network") LOG.info("Creating Host-only Network")
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'create'], subprocess.check_output(
stderr=subprocess.STDOUT) ["vboxmanage", "hostonlyif", "create"], stderr=subprocess.STDOUT
)
LOG.info("Provisioning %s with IP %s and Netmask %s", name, ip, netmask) LOG.info("Provisioning %s with IP %s and Netmask %s", name, oam_ip, netmask)
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'ipconfig', name, '--ip', subprocess.check_output(
ip, '--netmask', netmask], stderr=subprocess.STDOUT) [
"vboxmanage",
"hostonlyif",
"ipconfig",
name,
"--ip",
oam_ip,
"--netmask",
netmask,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_hostonlyifdelete(name="vboxnet0"): def vboxmanage_hostonlyifdelete(name="vboxnet0"):
""" """
Deletes hostonly network. This is used as a work around for creating too many hostonlyifs. Deletes hostonly network. This is used as a work around for creating too many hostonlyifs.
""" """
assert name, "Must provide network name" assert name, "Must provide network name"
LOG.info("Removing Host-only Network") LOG.info("Removing Host-only Network")
result = subprocess.check_output(['vboxmanage', 'hostonlyif', 'remove', name], subprocess.check_output(
stderr=subprocess.STDOUT) ["vboxmanage", "hostonlyif", "remove", name], stderr=subprocess.STDOUT
)
def vboxmanage_modifyvm(hostname=None, cpus=None, memory=None, nic=None, def vboxmanage_modifyvm(hostname, vm_config=None):
nictype=None, nicpromisc=None, nicnum=None,
intnet=None, hostonlyadapter=None,
natnetwork=None, uartbase=None, uartport=None,
uartmode=None, uartpath=None, nicbootprio2=1, prefix=""):
""" """
This modifies a VM with a specified name. Modify a virtual machine according to a specified configuration.
Args:
hostname(str): Name of host to modify
vm_config (dict): A dictionary representing the configuration options
for the virtual machine. Possible key values: cpus, memory, nic, nictype,
nicpromisc, nicnum, intnet, hostonlyadapter, natnetwork, uartbase,
uartport, uartmode, uartpath, nicbootprio2=1, prefix=""
Returns:
None
""" """
assert hostname, "Hostname is required" #put default values in nicbootprio2 and prefix if they not exist
# Add more semantic checks vm_config["nicbootprio2"] = vm_config.get("nicbootprio2", 1)
cmd = ['vboxmanage', 'modifyvm', hostname] vm_config["prefix"] = vm_config.get("prefix", "")
if cpus:
cmd.extend(['--cpus', cpus]) cmd = ["vboxmanage", "modifyvm", hostname]
if memory: nic_cmd = []
cmd.extend(['--memory', memory])
if nic and nictype and nicpromisc and nicnum: if _contains_value("cpus", vm_config):
cmd.extend(['--nic{}'.format(nicnum), nic]) cmd.extend(["--cpus", vm_config["cpus"]])
cmd.extend(['--nictype{}'.format(nicnum), nictype])
cmd.extend(['--nicpromisc{}'.format(nicnum), nicpromisc]) if _contains_value("memory", vm_config):
if intnet: cmd.extend(["--memory", vm_config["memory"]])
if prefix:
intnet = "{}-{}".format(prefix, intnet) if _is_network_configured(vm_config):
else: nic_cmd = _get_network_configuration(vm_config)
intnet = "{}".format(intnet) cmd.extend(nic_cmd)
cmd.extend(['--intnet{}'.format(nicnum), intnet])
if hostonlyadapter: elif _is_nat_network_configured(vm_config):
cmd.extend(['--hostonlyadapter{}'.format(nicnum), hostonlyadapter]) cmd.extend([f'--nic{vm_config["nicnum"]}', "nat"])
if natnetwork:
cmd.extend(['--nat-network{}'.format(nicnum), natnetwork]) if _is_uart_configured(vm_config):
elif nicnum and nictype == 'nat': uart_config = _add_uart(hostname, vm_config)
cmd.extend(['--nic{}'.format(nicnum), 'nat']) cmd.extend(uart_config)
if uartbase and uartport and uartmode and uartpath:
cmd.extend(['--uart1']) if _contains_value("nicbootprio2", vm_config):
cmd.extend(['{}'.format(uartbase)]) cmd.extend(["--nicbootprio2"])
cmd.extend(['{}'.format(uartport)]) cmd.extend([f'{vm_config["nicbootprio2"]}'])
cmd.extend(['--uartmode1'])
cmd.extend(['{}'.format(uartmode)]) cmd.extend(["--boot4"])
if platform == 'win32' or platform == 'win64': cmd.extend(["net"])
cmd.extend(['{}'.format(env.PORT)])
env.PORT += 1
else:
if prefix:
prefix = "{}_".format(prefix)
if 'controller-0' in hostname:
cmd.extend(['{}{}{}_serial'.format(uartpath, prefix, hostname)])
else:
cmd.extend(['{}{}{}'.format(uartpath, prefix, hostname)])
if nicbootprio2:
cmd.extend(['--nicbootprio2'])
cmd.extend(['{}'.format(nicbootprio2)])
cmd.extend(['--boot4'])
cmd.extend(['net'])
LOG.info(cmd) LOG.info(cmd)
LOG.info("Updating VM %s configuration", hostname) LOG.info("Updating VM %s configuration", hostname)
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def _is_network_configured(vm_config):
"""
Checks whether a network interface is configured in the given VM configuration.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
bool: True if a network interface is configured, False otherwise.
"""
return (_contains_value("nic", vm_config)
and _contains_value("nictype", vm_config)
and _contains_value("nicpromisc", vm_config)
and _contains_value("nicnum", vm_config)
)
def _get_network_configuration(vm_config):
"""
Constructs a list of options for the network interface based on the values in vm_config.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
list: A list of command-line options for the network interface.
"""
nic_cmd = [f'--nic{vm_config["nicnum"]}', vm_config["nic"]]
nic_cmd.extend([f'--nictype{vm_config["nicnum"]}', vm_config["nictype"]])
nic_cmd.extend([f'--nicpromisc{vm_config["nicnum"]}', vm_config["nicpromisc"]])
if _contains_value("intnet", vm_config):
intnet = vm_config["intnet"]
if _contains_value("prefix", vm_config):
intnet = f"{vm_config['prefix']}-{intnet}"
else:
intnet = f"{intnet}"
nic_cmd.extend([f'--intnet{vm_config["nicnum"]}', intnet])
if _contains_value("hostonlyadapter", vm_config):
nic_cmd.extend(
[
f'--hostonlyadapter{vm_config["nicnum"]}',
vm_config["hostonlyadapter"],
]
)
if _contains_value("natnetwork", vm_config):
nic_cmd.extend(
[f'--nat-network{vm_config["nicnum"]}', vm_config["natnetwork"]]
)
return nic_cmd
def _is_nat_network_configured(vm_config):
"""
Checks whether the NAT network is configured in the given VM configuration.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
bool: True if the NAT network is configured, False otherwise.
"""
return _contains_value("nicnum", vm_config) and vm_config.get("nictype") == "nat"
def _is_uart_configured(vm_config):
"""
Checks whether the UART device is configured in the given VM configuration.
Args:
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
bool: True if the UART device is configured, False otherwise.
"""
return (
_contains_value("uartbase", vm_config)
and _contains_value("uartport", vm_config)
and _contains_value("uartmode", vm_config)
and _contains_value("uartpath", vm_config)
)
def _add_uart(hostname, vm_config):
"""
Constructs a list of options for the UART device based on the values in vm_config.
Args:
hostname (str): Name of the virtual machine.
vm_config (dict): A dictionary representing the configuration options for the VM.
Returns:
list: A list of command-line options for the UART device.
"""
uart_config = ["--uart1"]
uart_config.extend([f'{vm_config["uartbase"]}'])
uart_config.extend([f'{vm_config["uartport"]}'])
uart_config.extend(["--uartmode1"])
uart_config.extend([f'{vm_config["uartmode"]}'])
if platform in ("win32", "win64"):
uart_config.extend([f"{env.PORT}"])
env.PORT += 1
else:
if _contains_value("prefix", vm_config):
prefix = f'{vm_config["prefix"]}_'
if "controller-0" in hostname:
uart_config.extend([f'{vm_config["uartpath"]}{prefix}{hostname}_serial'])
else:
uart_config.extend([f'{vm_config["uartpath"]}{prefix}{hostname}'])
return uart_config
def _contains_value(key, dictionary):
return key in dictionary and dictionary[key]
def vboxmanage_port_forward(hostname, network, local_port, guest_port, guest_ip): def vboxmanage_port_forward(hostname, network, local_port, guest_port, guest_ip):
"""
Configures port forwarding for a NAT network in VirtualBox.
Args:
hostname (str): Name of the virtual machine.
network (str): Name of the NAT network.
local_port (int): The local port number to forward.
guest_port (int): The port number on the guest to forward to.
guest_ip (str): The IP address of the guest to forward to.
Returns:
None
"""
# VBoxManage natnetwork modify --netname natnet1 --port-forward-4 # VBoxManage natnetwork modify --netname natnet1 --port-forward-4
# "ssh:tcp:[]:1022:[192.168.15.5]:22" # "ssh:tcp:[]:1022:[192.168.15.5]:22"
rule_name = "{}-{}".format(hostname, guest_port) rule_name = f"{hostname}-{guest_port}"
# Delete previous entry, if any # Delete previous entry, if any
LOG.info("Removing previous forwarding rule '%s' from NAT network '%s'", rule_name, network) LOG.info(
cmd = ['vboxmanage', 'natnetwork', 'modify', '--netname', network, "Removing previous forwarding rule '%s' from NAT network '%s'",
'--port-forward-4', 'delete', rule_name] rule_name,
network,
)
cmd = [
"vboxmanage",
"natnetwork",
"modify",
"--netname",
network,
"--port-forward-4",
"delete",
rule_name,
]
try: try:
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
pass pass
# Add new rule # Add new rule
rule = "{}:tcp:[]:{}:[{}]:{}".format(rule_name, local_port, guest_ip, guest_port) rule = f"{rule_name}:tcp:[]:{local_port}:[{guest_ip}]:{guest_port}"
LOG.info("Updating port-forwarding rule to: %s", rule) LOG.info("Updating port-forwarding rule to: %s", rule)
cmd = ['vboxmanage', 'natnetwork', 'modify', '--netname', network, '--port-forward-4', rule] cmd = [
result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) "vboxmanage",
"natnetwork",
"modify",
"--netname",
network,
"--port-forward-4",
rule,
]
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"): def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):
""" """
@ -317,62 +592,132 @@ def vboxmanage_storagectl(hostname=None, storectl="sata", hostiocache="off"):
assert hostname, "Hostname is required" assert hostname, "Hostname is required"
assert storectl, "Type of storage controller is required" assert storectl, "Type of storage controller is required"
LOG.info("Creating %s storage controller on VM %s", storectl, hostname) LOG.info("Creating %s storage controller on VM %s", storectl, hostname)
result = subprocess.check_output(['vboxmanage', 'storagectl', subprocess.check_output(
hostname, '--name', storectl, [
'--add', storectl, '--hostiocache', "vboxmanage",
hostiocache], stderr=subprocess.STDOUT) "storagectl",
hostname,
"--name",
storectl,
"--add",
storectl,
"--hostiocache",
hostiocache,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_storageattach(hostname=None, storectl="sata", def vboxmanage_storageattach(hostname, storage_config):
storetype="hdd", disk=None, port_num="0", device_num="0"):
""" """
This attaches a disk to a controller. Attaches a disk to a storage controller.
Args:
hostname (str): Name of the virtual machine.
storage_config (dict): A dictionary containing the config options for the storage device.
Possible key values: storectl, storetype, disk, port_num, device_num.
Returns:
str: The output of the VBoxManage command.
""" """
assert hostname, "Hostname is required" assert hostname, "Hostname is required"
assert storage_config and isinstance(storage_config, dict), "Storage configuration is required"
storectl = storage_config.get("storectl", "sata")
storetype = storage_config.get("storetype", "hdd")
disk = storage_config.get("disk")
port_num = storage_config.get("port_num", "0")
device_num = storage_config.get("device_num", "0")
assert disk, "Disk name is required" assert disk, "Disk name is required"
assert storectl, "Name of storage controller is required" assert storectl, "Name of storage controller is required"
assert storetype, "Type of storage controller is required" assert storetype, "Type of storage controller is required"
LOG.info("Attaching %s storage to storage controller %s on VM %s",
storetype, storectl, hostname)
result = subprocess.check_output(['vboxmanage', 'storageattach',
hostname, '--storagectl', storectl,
'--medium', disk, '--type',
storetype, '--port', port_num,
'--device', device_num], stderr=subprocess.STDOUT)
return result
def vboxmanage_deletemedium(hostname, vbox_home_dir='/home'): LOG.info(
"Attaching %s storage to storage controller %s on VM %s",
storetype,
storectl,
hostname,
)
return subprocess.check_output(
[
"vboxmanage",
"storageattach",
hostname,
"--storagectl",
storectl,
"--medium",
disk,
"--type",
storetype,
"--port",
port_num,
"--device",
device_num,
],
stderr=subprocess.STDOUT,
)
def vboxmanage_deletemedium(hostname, vbox_home_dir="/home"):
"""
Deletes the disk medium associated with a virtual machine.
Args:
hostname (str): The name of the virtual machine to which the disk medium is attached.
vbox_home_dir (str): The directory in which the disk medium files are stored.
Defaults to "/home".
Returns:
None
"""
assert hostname, "Hostname is required" assert hostname, "Hostname is required"
if platform == 'win32' or platform == 'win64': if platform in ("win32", "win64"):
return return
username = getpass.getuser() username = getpass.getuser()
vbox_home_dir = "{}/{}/vbox_disks/".format(vbox_home_dir, username) vbox_home_dir = f"{vbox_home_dir}/{username}/vbox_disks/"
disk_list = [f for f in os.listdir(vbox_home_dir) if disk_list = [
os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f] f
for f in os.listdir(vbox_home_dir)
if os.path.isfile(os.path.join(vbox_home_dir, f)) and hostname in f
]
LOG.info("Disk mediums to delete: %s", disk_list) LOG.info("Disk mediums to delete: %s", disk_list)
for disk in disk_list: for disk in disk_list:
LOG.info("Disconnecting disk %s from vbox.", disk) LOG.info("Disconnecting disk %s from vbox.", disk)
try: try:
result = subprocess.check_output(['vboxmanage', 'closemedium', 'disk', result = subprocess.check_output(
"{}{}".format(vbox_home_dir, disk), '--delete'], [
stderr=subprocess.STDOUT) "vboxmanage",
"closemedium",
"disk",
"{vbox_home_dir}{disk}",
"--delete",
],
stderr=subprocess.STDOUT,
)
LOG.info(result) LOG.info(result)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as exception:
# Continue if failures, disk may not be present # Continue if failures, disk may not be present
LOG.info("Error disconnecting disk, continuing. " LOG.info(
"Details: stdout: %s stderr: %s", e.stdout, e.stderr) "Error disconnecting disk, continuing. "
"Details: stdout: %s stderr: %s",
exception.stdout,
exception.stderr,
)
LOG.info("Removing backing file %s", disk) LOG.info("Removing backing file %s", disk)
try: try:
os.remove("{}{}".format(vbox_home_dir, disk)) os.remove("{vbox_home_dir}{disk}")
except: except: # pylint: disable=bare-except
pass pass
def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'): def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir="/home"):
""" """
This creates the required disks. This creates the required disks.
""" """
@ -385,28 +730,63 @@ def vboxmanage_createmedium(hostname=None, disk_list=None, vbox_home_dir='/home'
port_num = 0 port_num = 0
disk_count = 1 disk_count = 1
for disk in disk_list: for disk in disk_list:
if platform == 'win32' or platform == 'win64': if platform in ("win32", "win64"):
file_name = "C:\\Users\\" + username + "\\vbox_disks\\" + \ file_name = (
hostname + "_disk_{}".format(disk_count) "C:\\Users\\"
+ username
+ "\\vbox_disks\\"
+ hostname
+ f"_disk_{disk_count}"
)
else: else:
file_name = vbox_home_dir + '/' + username + "/vbox_disks/" \ file_name = (
+ hostname + "_disk_{}".format(disk_count) vbox_home_dir
LOG.info("Creating disk %s of size %s on VM %s on device %s port %s", + "/"
file_name, disk, hostname, device_num, port_num) + username
+ "/vbox_disks/"
+ hostname
+ f"_disk_{disk_count}"
)
LOG.info(
"Creating disk %s of size %s on VM %s on device %s port %s",
file_name,
disk,
hostname,
device_num,
port_num,
)
try: try:
result = subprocess.check_output(['vboxmanage', 'createmedium', result = subprocess.check_output(
'disk', '--size', str(disk), [
'--filename', file_name, "vboxmanage",
'--format', 'vdi', "createmedium",
'--variant', 'standard'], "disk",
stderr=subprocess.STDOUT) "--size",
str(disk),
"--filename",
file_name,
"--format",
"vdi",
"--variant",
"standard",
],
stderr=subprocess.STDOUT,
)
LOG.info(result) LOG.info(result)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as exception:
LOG.info("Error stdout: %s stderr: %s", e.stdout, e.stderr) LOG.info("Error stdout: %s stderr: %s", exception.stdout, exception.stderr)
raise raise
vboxmanage_storageattach(hostname, "sata", "hdd", file_name + \ vboxmanage_storageattach(
".vdi", str(port_num), str(device_num)) hostname,
{
"storectl": "sata",
"storetype": "hdd",
"disk": file_name + ".vdi",
"port_num": str(port_num),
"device_num": str(device_num),
},
)
disk_count += 1 disk_count += 1
port_num += 1 port_num += 1
time.sleep(5) time.sleep(5)
@ -425,12 +805,13 @@ def vboxmanage_startvm(hostname=None, force=False):
else: else:
running_vms = [] running_vms = []
if hostname.encode('utf-8') in running_vms: if hostname.encode("utf-8") in running_vms:
LOG.info("Host %s is already started", hostname) LOG.info("Host %s is already started", hostname)
else: else:
LOG.info("Powering on VM %s", hostname) LOG.info("Powering on VM %s", hostname)
result = subprocess.check_output(['vboxmanage', 'startvm', result = subprocess.check_output(
hostname], stderr=subprocess.STDOUT) ["vboxmanage", "startvm", hostname], stderr=subprocess.STDOUT
)
LOG.info(result) LOG.info(result)
# Wait for VM to start # Wait for VM to start
@ -438,11 +819,11 @@ def vboxmanage_startvm(hostname=None, force=False):
while tmout: while tmout:
tmout -= 1 tmout -= 1
running_vms = vboxmanage_list(option="runningvms") running_vms = vboxmanage_list(option="runningvms")
if hostname.encode('utf-8') in running_vms: if hostname.encode("utf-8") in running_vms:
break break
time.sleep(1) time.sleep(1)
else: else:
raise "Failed to start VM: {}".format(hostname) raise f"Failed to start VM: {hostname}"
LOG.info("VM '%s' started.", hostname) LOG.info("VM '%s' started.", hostname)
@ -456,8 +837,9 @@ def vboxmanage_controlvms(hosts=None, action=None):
for host in hosts: for host in hosts:
LOG.info("Executing %s action on VM %s", action, host) LOG.info("Executing %s action on VM %s", action, host)
result = subprocess.call(["vboxmanage", "controlvm", host, subprocess.call(
action], stderr=subprocess.STDOUT) ["vboxmanage", "controlvm", host, action], stderr=subprocess.STDOUT
)
time.sleep(1) time.sleep(1)
@ -471,8 +853,9 @@ def vboxmanage_takesnapshot(hosts=None, name=None):
for host in hosts: for host in hosts:
LOG.info("Taking snapshot %s on VM %s", name, host) LOG.info("Taking snapshot %s on VM %s", name, host)
result = subprocess.call(["vboxmanage", "snapshot", host, "take", subprocess.call(
name], stderr=subprocess.STDOUT) ["vboxmanage", "snapshot", host, "take", name], stderr=subprocess.STDOUT
)
def vboxmanage_restoresnapshot(host=None, name=None): def vboxmanage_restoresnapshot(host=None, name=None):
@ -484,7 +867,7 @@ def vboxmanage_restoresnapshot(host=None, name=None):
assert name, "Need to provide the snapshot to restore" assert name, "Need to provide the snapshot to restore"
LOG.info("Restoring snapshot %s on VM %s", name, host) LOG.info("Restoring snapshot %s on VM %s", name, host)
result = subprocess.call(["vboxmanage", "snapshot", host, "restore", subprocess.call(
name], stderr=subprocess.STDOUT) ["vboxmanage", "snapshot", host, "restore", name], stderr=subprocess.STDOUT
)
time.sleep(10) time.sleep(10)

1186
virtualbox/pybox/install_vbox.py
View File

File diff suppressed because it is too large Load Diff

1
virtualbox/pybox/requirements.txt
View File

@ -3,4 +3,5 @@ paramiko
pytest pytest
git+https://github.com/digidotcom/python-streamexpect#egg=streamexpect git+https://github.com/digidotcom/python-streamexpect#egg=streamexpect
pexpect pexpect
ruamel.yaml

42
virtualbox/pybox/utils/install_log.py
View File

@ -3,36 +3,45 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module provides functionality to initialize logging for a lab, create a sub-directory
for the current run, set the logging level, and create a symbolic link to the latest logs
for the lab.
"""
import os import os
import datetime import datetime
import logging import logging
from consts.env import LOGPATH from consts.env import LOGPATH
log_dir = "" LOG_DIR = ""
LOG = logging.getLogger() LOG = logging.getLogger()
def init_logging(lab_name, log_path=None): def init_logging(lab_name, log_path=None):
global LOG, log_dir """
This method initializes the logging for a lab. It creates a sub-directory for the
current run and sets the logging level to INFO. It also creates a symbolic link to
the latest logs for the lab. The method takes in the lab name and an optional log path
parameter. If no log path is specified, it uses the default path provided by the
LOGPATH constant in the env module.
"""
global LOG, LOG_DIR # pylint: disable=global-statement, global-variable-not-assigned
if not log_path: if not log_path:
log_path = LOGPATH log_path = LOGPATH
lab_log_path = log_path + "/" + lab_name lab_log_path = log_path + "/" + lab_name
# Setup log sub-directory for current run # Setup log sub-directory for current run
current_time = datetime.datetime.now() current_time = datetime.datetime.now()
log_dir = "{}/{}_{}_{}_{}_{}_{}".format(lab_log_path, LOG_DIR = f"{lab_log_path}/{current_time.year}_{current_time.month}_\
current_time.year, {current_time.day}_{current_time.hour}_{current_time.minute}_{current_time.second}"
current_time.month, if not os.path.exists(LOG_DIR):
current_time.day, os.makedirs(LOG_DIR)
current_time.hour,
current_time.minute,
current_time.second)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
LOG.setLevel(logging.INFO) LOG.setLevel(logging.INFO)
formatter = logging.Formatter("%(asctime)s: %(message)s") formatter = logging.Formatter("%(asctime)s: %(message)s")
log_file = "{}/install.log".format(log_dir) log_file = f"{LOG_DIR}/install.log"
handler = logging.FileHandler(log_file) handler = logging.FileHandler(log_file)
handler.setFormatter(formatter) handler.setFormatter(formatter)
handler.setLevel(logging.INFO) handler.setLevel(logging.INFO)
@ -44,9 +53,12 @@ def init_logging(lab_name, log_path=None):
# Create symbolic link to latest logs of this lab # Create symbolic link to latest logs of this lab
try: try:
os.unlink(lab_log_path + "/latest") os.unlink(lab_log_path + "/latest")
except: except: # pylint: disable=bare-except
pass pass
os.symlink(log_dir, lab_log_path + "/latest") os.symlink(LOG_DIR, lab_log_path + "/latest")
def get_log_dir(): def get_log_dir():
return log_dir """This method returns the directory path of the current logging run."""
return LOG_DIR

59
virtualbox/pybox/utils/kpi.py
View File

@ -3,55 +3,80 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module provides functions to track and report key performance indicators (KPIs) for a program.
"""
import time import time
from utils.install_log import LOG from utils.install_log import LOG
STAGES = [] STAGES = []
METRICS = {} METRICS = {}
start = 0 START = 0
def init_kpi_metrics(): def init_kpi_metrics():
global start """
start = time.time() Initializes the global variable START with the current time to start tracking the
duration of a program.
"""
global START # pylint: disable=global-statement
START = time.time()
def get_formated_time(sec): def get_formated_time(sec):
"""
Takes the duration in seconds and formats it in hours, minutes and seconds.
Returns the formatted string.
"""
hours = sec // 3600 hours = sec // 3600
sec %= 3600 sec %= 3600
minutes = sec // 60 minutes = sec // 60
sec %= 60 sec %= 60
seconds = sec seconds = sec
if hours: if hours:
return "{:.0f}h {:.0f}m {:.2f}s".format(hours, minutes, seconds) return f"{hours:.0f}h {minutes:.0f}m {seconds:.2f}s"
elif minutes: if minutes:
return "{:.0f}m {:.2f}s".format(minutes, seconds) return f"{minutes:.0f}m {seconds:.2f}s"
elif seconds: return f"{seconds:.2f}s"
return "{:.2f}s".format(seconds)
def set_kpi_metric(metric, duration): def set_kpi_metric(metric, duration):
global METRICS, STAGES """Sets the duration of a metric and adds the metric to the global list of STAGES."""
global METRICS, STAGES # pylint: disable=global-statement, global-variable-not-assigned
METRICS[metric] = duration METRICS[metric] = duration
STAGES.append(metric) STAGES.append(metric)
def print_kpi(metric): def print_kpi(metric):
"""Takes a metric as input and prints the duration of that metric using the LOG module."""
if metric in STAGES: if metric in STAGES:
sec = METRICS[metric] sec = METRICS[metric]
LOG.info(" Time in stage '%s': %s ", metric, get_formated_time(sec)) LOG.info(" Time in stage '%s': %s ", metric, get_formated_time(sec))
elif metric == 'total' and start: elif metric == 'total' and START:
duration = time.time() - start duration = time.time() - START
LOG.info(" Total time: %s", get_formated_time(duration)) LOG.info(" Total time: %s", get_formated_time(duration))
def get_kpi_str(metric): def get_kpi_str(metric):
"""Takes a metric as input and returns the duration of that metric as a formatted string."""
msg = "" msg = ""
if metric in STAGES: if metric in STAGES:
sec = METRICS[metric] sec = METRICS[metric]
msg += (" Time in stage '{}': {} \n".format(metric, get_formated_time(sec))) msg += (f" Time in stage '{metric}': {get_formated_time(sec)} \n")
elif metric == 'total' and start: elif metric == 'total' and START:
duration = time.time() - start duration = time.time() - START
msg += (" Total time: {}\n".format(get_formated_time(duration))) msg += (f" Total time: {get_formated_time(duration)}\n")
return msg return msg
def get_kpi_metrics_str(): def get_kpi_metrics_str():
"""Returns a formatted string with all the metrics and their durations."""
msg = "===================== Metrics ====================\n" msg = "===================== Metrics ====================\n"
for stage in STAGES: for stage in STAGES:
msg += get_kpi_str(stage) msg += get_kpi_str(stage)
@ -59,10 +84,12 @@ def get_kpi_metrics_str():
msg += "===============================================\n" msg += "===============================================\n"
return msg return msg
def print_kpi_metrics(): def print_kpi_metrics():
"""Prints all the metrics and their durations using the LOG module."""
LOG.info("===================== Metrics ====================") LOG.info("===================== Metrics ====================")
for stage in STAGES: for stage in STAGES:
print_kpi(stage) print_kpi(stage)
print_kpi('total') print_kpi('total')
LOG.info("==================================================") LOG.info("==================================================")

107
virtualbox/pybox/utils/serial.py
View File

@ -3,6 +3,10 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module provides functionality to connect and communicate with a remote host
using local domain socket.
"""
import re import re
import socket import socket
@ -22,27 +26,27 @@ def connect(hostname, port=10000, prefix=""):
""" """
if prefix: if prefix:
prefix = "{}_".format(prefix) prefix = f"{prefix}_"
socketname = "/tmp/{}{}".format(prefix, hostname) socketname = f"/tmp/{prefix}{hostname}"
if 'controller-0'in hostname: if 'controller-0' in hostname:
socketname += '_serial' socketname += '_serial'
LOG.info("Connecting to %s at %s", hostname, socketname) LOG.info("Connecting to %s at %s", hostname, socketname)
if platform == 'win32' or platform == 'win64': if platform in ('win32', 'win64'):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP)
else: else:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try: try:
if platform == 'win32' or platform == 'win64': if platform in ('win32', 'win64'):
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.connect(('localhost', port)) sock.connect(('localhost', port))
else: else:
sock.connect(socketname) sock.connect(socketname)
except: except: # pylint: disable=bare-except
LOG.info("Connection failed") LOG.info("Connection failed")
pass pass # pylint: disable=unnecessary-pass
# disconnect(sock) # disconnect(sock)
sock = None sock = None
# TODO (WEI): double check this # TODO (WEI): double check this # pylint: disable=fixme
sock.setblocking(0) sock.setblocking(0)
return sock return sock
@ -61,14 +65,18 @@ def disconnect(sock):
sock.shutdown(socket.SHUT_RDWR) sock.shutdown(socket.SHUT_RDWR)
sock.close() sock.close()
def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, flush=True):
#TODO: Not testested, will not work if kernel or other processes throw data on stdout or stderr # pylint: disable=too-many-arguments, too-many-locals, too-many-branches
def get_output(stream, prompts=None, timeout=5, log=True, as_lines=True, flush=True):
# pylint: disable=fixme
# TODO: Not tested, will not work if kernel or other processes throw data on stdout or stderr
""" """
Execute a command and get its output. Make sure no other command is executing. Execute a command and get its output. Make sure no other command is executing.
And 'dmesg -D' was executed. And 'dmesg -D' was executed.
""" """
POLL_PERIOD = 0.1
MAX_READ_BUFFER = 1024 poll_period = 0.1
max_read_buffer = 1024
data = "" data = ""
line_buf = "" line_buf = ""
lines = [] lines = []
@ -82,13 +90,13 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
try: try:
LOG.info("Buffer has bytes before cmd execution: %s", LOG.info("Buffer has bytes before cmd execution: %s",
trash.decode('utf-8')) trash.decode('utf-8'))
except Exception: except Exception: # pylint: disable=W0703
pass pass
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
pass pass
# Send command # Send command
stream.sendall("{}\n".format(cmd).encode('utf-8')) stream.sendall("{cmd}\n".encode('utf-8'))
# Get response # Get response
patterns = [] patterns = []
@ -98,20 +106,21 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
now = time.time() now = time.time()
end_time = now + float(timeout) end_time = now + float(timeout)
prev_timeout = stream.gettimeout() prev_timeout = stream.gettimeout()
stream.settimeout(POLL_PERIOD) stream.settimeout(poll_period)
incoming = None incoming = None
# pylint: disable=too-many-nested-blocks
try: try:
while (end_time - now) >= 0: while (end_time - now) >= 0:
try: try:
incoming = stream.recv(MAX_READ_BUFFER) incoming = stream.recv(max_read_buffer)
except socket.timeout: except socket.timeout:
pass pass
if incoming: if incoming:
data += incoming data += incoming
if log: if log:
for c in incoming: for char in incoming:
if c != '\n': if char != '\n':
line_buf += c line_buf += char
else: else:
LOG.info(line_buf) LOG.info(line_buf)
lines.append(line_buf) lines.append(line_buf)
@ -120,8 +129,7 @@ def get_output(stream, cmd, prompts=None, timeout=5, log=True, as_lines=True, fl
if pattern.search(data): if pattern.search(data):
if as_lines: if as_lines:
return lines return lines
else: return data
return data
now = time.time() now = time.time()
raise streamexpect.ExpectTimeout() raise streamexpect.ExpectTimeout()
finally: finally:
@ -132,23 +140,24 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
""" """
Wait for user specified text from stream. Wait for user specified text from stream.
""" """
time.sleep(1) time.sleep(1)
if timeout < 60: if timeout < 60:
LOG.info("Expecting text within %s seconds: %s\n", timeout, text) LOG.info("Expecting text within %s seconds: %s\n", timeout, text)
else: else:
LOG.info("Expecting text within %s minutes: %s\n", timeout/60, text) LOG.info("Expecting text within %s minutes: %s\n", timeout / 60, text)
try: try:
stream.expect_bytes("{}".format(text).encode('utf-8'), timeout=timeout) stream.expect_bytes(f"{text}".encode('utf-8'), timeout=timeout)
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
if fail_ok: if fail_ok:
return -1 return -1
else:
stdout.write('\n') stdout.write('\n')
LOG.error("Did not find expected text") LOG.error("Did not find expected text")
# disconnect(stream) # disconnect(stream)
raise raise
except Exception as e: except Exception as exception:
LOG.info("Connection failed with %s", e) LOG.info("Connection failed with %s", exception)
raise raise
stdout.write('\n') stdout.write('\n')
@ -162,8 +171,8 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
incoming += b'\n' incoming += b'\n'
try: try:
LOG.info(">>> expect_bytes: Buffer has bytes!") LOG.info(">>> expect_bytes: Buffer has bytes!")
stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it
except Exception: except Exception: # pylint: disable=W0703
pass pass
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
pass pass
@ -171,11 +180,13 @@ def expect_bytes(stream, text, timeout=180, fail_ok=False, flush=True):
return 0 return 0
# pylint: disable=inconsistent-return-statements
def send_bytes(stream, text, fail_ok=False, expect_prompt=True, def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
prompt=None, timeout=180, send=True, flush=True): prompt=None, timeout=180, send=True, flush=True):
""" """
Send user specified text to stream. Send user specified text to stream.
""" """
time.sleep(1) time.sleep(1)
if flush: if flush:
try: try:
@ -184,8 +195,8 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
incoming += b'\n' incoming += b'\n'
try: try:
LOG.info(">>> send_bytes: Buffer has bytes!") LOG.info(">>> send_bytes: Buffer has bytes!")
stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it stdout.write(incoming.decode('utf-8')) # streamexpect hardcodes it
except Exception: except Exception: # pylint: disable=W0703
pass pass
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
pass pass
@ -193,28 +204,28 @@ def send_bytes(stream, text, fail_ok=False, expect_prompt=True,
LOG.info("Sending text: %s", text) LOG.info("Sending text: %s", text)
try: try:
if send: if send:
stream.sendall("{}\n".format(text).encode('utf-8')) stream.sendall(f"{text}\n".encode('utf-8'))
else: else:
stream.sendall("{}".format(text).encode('utf-8')) stream.sendall(f"{text}".encode('utf-8'))
if expect_prompt: if expect_prompt:
time.sleep(1) time.sleep(1)
if prompt: if prompt:
return expect_bytes(stream, prompt, timeout=timeout, fail_ok=fail_ok) return expect_bytes(stream, prompt, timeout=timeout, fail_ok=fail_ok)
else:
rc = expect_bytes(stream, "~$", timeout=timeout, fail_ok=True) return_code = expect_bytes(stream, "~$", timeout=timeout, fail_ok=True)
if rc != 0: if return_code != 0:
send_bytes(stream, '\n', expect_prompt=False) send_bytes(stream, '\n', expect_prompt=False)
expect_bytes(stream, 'keystone', timeout=timeout) expect_bytes(stream, 'keystone', timeout=timeout)
return return
except streamexpect.ExpectTimeout: except streamexpect.ExpectTimeout:
if fail_ok: if fail_ok:
return -1 return -1
else:
LOG.error("Failed to send text, logging out.") LOG.error("Failed to send text, logging out.")
stream.sendall("exit".encode('utf-8')) stream.sendall("exit".encode('utf-8'))
raise raise
except Exception as e: except Exception as exception:
LOG.info("Connection failed with %s.", e) LOG.info("Connection failed with %s.", exception)
raise raise
return 0 return 0

93
virtualbox/pybox/utils/sftp.py
View File

@ -3,6 +3,10 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# #
"""
This module provides functionality to send files and directories to remote servers using the
rsync and paramiko libraries.
"""
import getpass import getpass
import os import os
@ -12,10 +16,14 @@ import paramiko
from utils.install_log import LOG from utils.install_log import LOG
def sftp_send(source, remote_host, remote_port, destination, username, password): def sftp_send(source, destination, client_dict):
""" """
Send files to remote server Send files to remote server
""" """
remote_host = client_dict["remote_host"]
username = client_dict["username"]
LOG.info("Connecting to server %s with username %s", remote_host, username) LOG.info("Connecting to server %s with username %s", remote_host, username)
ssh_client = paramiko.SSHClient() ssh_client = paramiko.SSHClient()
@ -25,12 +33,17 @@ def sftp_send(source, remote_host, remote_port, destination, username, password)
retry = 0 retry = 0
while retry < 8: while retry < 8:
try: try:
ssh_client.connect(remote_host, port=remote_port, ssh_client.connect(
username=username, password=password, remote_host,
look_for_keys=False, allow_agent=False) port=client_dict["remote_port"],
username=username,
password=client_dict["password"],
look_for_keys=False,
allow_agent=False
)
sftp_client = ssh_client.open_sftp() sftp_client = ssh_client.open_sftp()
retry = 8 retry = 8
except Exception as e: except Exception: # pylint: disable=W0703
LOG.info("******* try again") LOG.info("******* try again")
retry += 1 retry += 1
time.sleep(10) time.sleep(10)
@ -42,8 +55,41 @@ def sftp_send(source, remote_host, remote_port, destination, username, password)
ssh_client.close() ssh_client.close()
def send_dir(source, remote_host, remote_port, destination, username, # pylint: disable=R0801
password, follow_links=True, clear_known_hosts=True): def send_dir(params_dict):
"""
Send directory `source` to remote host `remote_host` at port `remote_port` and destination
`destination` using `rsync` over `ssh`.
Args:
params_dict (dict): A dictionary containing the following keys:
- source (str): The local directory to be sent.
- remote_host (str): The IP address or domain name of the remote host.
- remote_port (int): The port number of the remote host to connect to.
- destination (str): The remote directory to copy `source` into.
- username (str): The username for the SSH connection.
- password (str): The password for the SSH connection.
- follow_links (bool, optional): Whether or not to follow symbolic links when
copying files. Default is True.
- clear_known_hosts (bool, optional): Whether or not to clear the known_hosts file
before making the SSH connection. Default is True.
Raises:
Exception: If there is an error in `rsync`, raises an exception with the return code.
Note:
This method only works from a Linux environment.
"""
source = params_dict['source']
remote_host = params_dict['remote_host']
remote_port = params_dict['remote_port']
destination = params_dict['destination']
username = params_dict['username']
password = params_dict['password']
follow_links = params_dict.get('follow_links', True)
clear_known_hosts = params_dict.get('clear_known_hosts', True)
# Only works from linux for now # Only works from linux for now
if not source.endswith('/') or not source.endswith('\\'): if not source.endswith('/') or not source.endswith('\\'):
source = source + '/' source = source + '/'
@ -51,29 +97,28 @@ def send_dir(source, remote_host, remote_port, destination, username,
follow_links = "L" if follow_links else "" follow_links = "L" if follow_links else ""
if clear_known_hosts: if clear_known_hosts:
if remote_host == '127.0.0.1': if remote_host == '127.0.0.1':
keygen_arg = "[127.0.0.1]:{}".format(remote_port) keygen_arg = f"[127.0.0.1]:{remote_port}"
else: else:
keygen_arg = remote_host keygen_arg = remote_host
cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}' cmd = f'ssh-keygen -f "/home/{getpass.getuser()}/.ssh/known_hosts" -R {keygen_arg}'
LOG.info("CMD: %s", cmd) LOG.info("CMD: %s", cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
for line in iter(process.stdout.readline, b''): for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip()) LOG.info("%s", line.decode("utf-8").strip())
process.wait() process.wait()
LOG.info(f'Running rsync of dir: {source} -> {username}@{remote_host}' LOG.info('Running rsync of dir: %s -> %s@%s:%s', source, username, remote_host, destination)
f':{destination}')
cmd = (f'rsync -av{follow_links} --rsh="/usr/bin/sshpass -p {password} ' cmd = (f'rsync -av{follow_links} --rsh="/usr/bin/sshpass -p {password} '
f'ssh -p {remote_port} -o StrictHostKeyChecking=no -l {username}" ' f'ssh -p {remote_port} -o StrictHostKeyChecking=no -l {username}" '
f'{source}* {username}@{remote_host}:{destination}') f'{source}* {username}@{remote_host}:{destination}')
LOG.info("CMD: %s", cmd) LOG.info("CMD: %s", cmd)
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) with subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) as process:
for line in iter(process.stdout.readline, b''): for line in iter(process.stdout.readline, b''):
LOG.info("%s", line.decode("utf-8").strip()) LOG.info("%s", line.decode("utf-8").strip())
process.wait() process.wait()
if process.returncode: if process.returncode:
raise Exception(f'Error in rsync, return code: {process.returncode}') raise Exception(f"Error in rsync, return code:{process.returncode}") # pylint: disable=E0012, W0719
def send_dir_fallback(source, remote_host, destination, username, password): def send_dir_fallback(source, remote_host, destination, username, password):
@ -87,10 +132,17 @@ def send_dir_fallback(source, remote_host, destination, username, password):
e.g. myhost.com e.g. myhost.com
- destination: where to store the file on host: /home/myuser/ - destination: where to store the file on host: /home/myuser/
""" """
LOG.info("Connecting to server %s with username %s", remote_host, username) LOG.info("Connecting to server %s with username %s", remote_host, username)
ssh_client = paramiko.SSHClient() ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(remote_host, username=username, password=password, look_for_keys=False, allow_agent=False) ssh_client.connect(
remote_host,
username=username,
password=password,
look_for_keys=False,
allow_agent=False
)
sftp_client = ssh_client.open_sftp() sftp_client = ssh_client.open_sftp()
path = '' path = ''
send_img = False send_img = False
@ -113,4 +165,3 @@ def send_dir_fallback(source, remote_host, destination, username, password):
ssh_client.close() ssh_client.close()
if send_img: if send_img:
time.sleep(10) time.sleep(10)