diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..6275299 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..da812ec --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/plugin-architecture.iml b/.idea/plugin-architecture.iml new file mode 100644 index 0000000..eec6694 --- /dev/null +++ b/.idea/plugin-architecture.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..0c3090f --- /dev/null +++ b/app.py @@ -0,0 +1,46 @@ +import argparse + +from engine import PluginEngine +from util import FileSystem + + +def __description() -> str: + return "Create your own anime meta data" + + +def __usage() -> str: + return "vrv-meta.py --service vrv" + + +def __init_cli() -> argparse: + parser = argparse.ArgumentParser(description=__description(), usage=__usage()) + parser.add_argument( + '-l', '--log', default='DEBUG', help=""" + Specify log level which should use. Default will always be DEBUG, choose between the following options + CRITICAL, ERROR, WARNING, INFO, DEBUG + """ + ) + parser.add_argument( + '-d', '--directory', default=f'{FileSystem.get_plugins_directory()}', help=""" + (Optional) Supply a directory where plugins should be loaded from. The default is ./plugins + """ + ) + return parser + + +def __print_program_end() -> None: + print("-----------------------------------") + print("End of execution") + print("-----------------------------------") + + +def __init_app(parameters: dict) -> None: + PluginEngine(options=parameters).start() + + +if __name__ == '__main__': + __cli_args = __init_cli().parse_args() + __init_app({ + 'log_level': __cli_args.log, + 'directory': __cli_args.directory + }) diff --git a/engine/__init__.py b/engine/__init__.py new file mode 100644 index 0000000..ec9fae9 --- /dev/null +++ b/engine/__init__.py @@ -0,0 +1,2 @@ +from .engine_contract import PluginCore, IPluginRegistry +from .engine_core import PluginEngine diff --git a/engine/engine_contract.py b/engine/engine_contract.py new file mode 100644 index 0000000..91316da --- /dev/null +++ b/engine/engine_contract.py @@ -0,0 +1,36 @@ +from logging import Logger +from typing import Optional, List + +from model import Meta, Device + + +class IPluginRegistry(type): + plugin_registries: List[type] = list() + + def __init__(cls, name, bases, attrs): + super().__init__(cls) + if name != 'PluginCore': + IPluginRegistry.plugin_registries.append(cls) + + +class PluginCore(object, metaclass=IPluginRegistry): + """ + Plugin core class + """ + + meta: Optional[Meta] + + def __init__(self, logger: Logger) -> None: + """ + Entry init block for plugins + :param logger: logger that plugins can make use of + """ + self._logger = logger + + def invoke(self, **args) -> Device: + """ + Starts main plugin flow + :param args: possible arguments for the plugin + :return: a device for the plugin + """ + pass diff --git a/engine/engine_core.py b/engine/engine_core.py new file mode 100644 index 0000000..f88c216 --- /dev/null +++ b/engine/engine_core.py @@ -0,0 +1,31 @@ +from logging import Logger + +from usecase import PluginUseCase +from util import LogUtil + + +class PluginEngine: + _logger: Logger + + def __init__(self, **args) -> None: + self._logger = LogUtil.create(args['options']['log_level']) + self.use_case = PluginUseCase(args['options']) + + def start(self) -> None: + self.__reload_plugins() + self.__invoke_on_plugins('Q') + + def __reload_plugins(self) -> None: + """Reset the list of all plugins and initiate the walk over the main + provided plugin package to load all available plugins + """ + self.use_case.discover_plugins(True) + + def __invoke_on_plugins(self, command: chr): + """Apply all of the plugins on the argument supplied to this function + """ + for module in self.use_case.modules: + plugin = self.use_case.register_plugin(module, self._logger) + delegate = self.use_case.hook_plugin(plugin) + device = delegate(command=command) + self._logger.info(f'Loaded device: {device}') diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000..f6014f0 --- /dev/null +++ b/model/__init__.py @@ -0,0 +1 @@ +from .models import Meta, Device, PluginConfig, DependencyModule, PluginRunTimeOption diff --git a/model/models.py b/model/models.py new file mode 100644 index 0000000..ff22f9a --- /dev/null +++ b/model/models.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class PluginRunTimeOption(object): + main: str + tests: Optional[List[str]] + + +@dataclass +class DependencyModule: + name: str + version: str + + def __str__(self) -> str: + return f'{self.name}=={self.version}' + + +@dataclass +class PluginConfig: + name: str + alias: str + creator: str + runtime: PluginRunTimeOption + repository: str + description: str + version: str + requirements: Optional[List[DependencyModule]] + + +@dataclass +class Meta: + name: str + description: str + version: str + + def __str__(self) -> str: + return f'{self.name}: {self.version}' + + +@dataclass +class Device: + name: str + firmware: int + protocol: str + errors: List[int] diff --git a/plugins/advanced-plugin/main.py b/plugins/advanced-plugin/main.py new file mode 100644 index 0000000..087dc07 --- /dev/null +++ b/plugins/advanced-plugin/main.py @@ -0,0 +1,57 @@ +from logging import Logger +from random import randint +from time import sleep +from typing import Optional + +from engine import PluginCore +from model import Meta, Device + + +class AdvanceSamplePlugin(PluginCore): + + def __init__(self, logger: Logger) -> None: + super().__init__(logger) + self.meta = Meta( + name='Advanced Sample Plugin', + description='Advanced Sample plugin template', + version='0.0.1' + ) + + @staticmethod + def __simulate_operation() -> None: + sleep_duration = randint(1, 100) / 100 + sleep(sleep_duration) + + def __get_firmware(self) -> int: + self._logger.debug('Enquiring device firmware') + self.__simulate_operation() + return 0xf41c3e + + def __get_protocol(self) -> str: + self._logger.debug('Enquiring messaging protocol') + self.__simulate_operation() + return "ASCII" + + def __get_errors(self) -> [int]: + self._logger.debug('Enquiring device errors') + self.__simulate_operation() + return [0x2f3a6c, 0xa8e1f5] + + def __create_device(self) -> Device: + firmware = self.__get_firmware() + protocol = self.__get_protocol() + errors = self.__get_errors() + + return Device( + name='Advanced Sample Device', + firmware=firmware, + protocol=protocol, + errors=errors + ) + + def invoke(self, command: chr, protocol: Optional[str] = None) -> Device: + self._logger.debug(f'Command: {command} -> {self.meta}') + device = self.__create_device() + if device.protocol != protocol: + self._logger.warning(f'Device does not support protocol supplied protocol') + return device diff --git a/plugins/advanced-plugin/plugin.yaml b/plugins/advanced-plugin/plugin.yaml new file mode 100644 index 0000000..9fe85bc --- /dev/null +++ b/plugins/advanced-plugin/plugin.yaml @@ -0,0 +1,15 @@ +name: 'Advanced Plugin' +alias: 'advanced-plugin' +creator: 'wax911' +runtime: + main: 'main.py' + tests: + - 'tests/core.py' +repository: 'https://github.com/wax911/advanced-plugin' +description: 'Advanced sample-plugin plugin template' +version: '0.0.1' +requirements: + - name: 'PyYAML' + version: '5.3.1' + - name: 'pytz' + version: '2019.3' \ No newline at end of file diff --git a/plugins/advanced-plugin/test/__init__.py b/plugins/advanced-plugin/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/advanced-plugin/test/core.py b/plugins/advanced-plugin/test/core.py new file mode 100644 index 0000000..58e5dfe --- /dev/null +++ b/plugins/advanced-plugin/test/core.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/plugins/sample-plugin/plugin.yaml b/plugins/sample-plugin/plugin.yaml new file mode 100644 index 0000000..fbf74ab --- /dev/null +++ b/plugins/sample-plugin/plugin.yaml @@ -0,0 +1,15 @@ +name: 'Sample Plugin' +alias: 'sample-plugin' +creator: 'wax911' +runtime: + main: 'sample.py' + tests: + - 'tests/core.py' +repository: 'https://github.com/wax911/sample-plugin' +description: 'Sample plugin template' +version: '0.0.2' +requirements: + - name: 'PyYAML' + version: '5.3.1' + - name: 'pytz' + version: '2019.3' \ No newline at end of file diff --git a/plugins/sample-plugin/sample.py b/plugins/sample-plugin/sample.py new file mode 100644 index 0000000..89c08a1 --- /dev/null +++ b/plugins/sample-plugin/sample.py @@ -0,0 +1,29 @@ +from logging import Logger + +from engine import PluginCore +from model import Meta, Device + + +class SamplePlugin(PluginCore): + + def __init__(self, logger: Logger) -> None: + super().__init__(logger) + self.meta = Meta( + name='Sample Plugin', + description='Sample plugin template', + version='0.0.1' + ) + + @staticmethod + def __create_device() -> Device: + return Device( + name='Sample Device', + firmware=0xa2c3f, + protocol='SAMPLE', + errors=[0x0000] + ) + + def invoke(self, command: chr) -> Device: + self._logger.debug(f'Command: {command} -> {self.meta}') + device = self.__create_device() + return device diff --git a/plugins/sample-plugin/test/__init__.py b/plugins/sample-plugin/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/sample-plugin/test/core.py b/plugins/sample-plugin/test/core.py new file mode 100644 index 0000000..58e5dfe --- /dev/null +++ b/plugins/sample-plugin/test/core.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) + + +if __name__ == '__main__': + unittest.main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a6105ad --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +PyYAML==5.3.1 +packaging==20.4 +dacite==1.5.0 \ No newline at end of file diff --git a/settings/configuration.yaml b/settings/configuration.yaml new file mode 100644 index 0000000..b81031a --- /dev/null +++ b/settings/configuration.yaml @@ -0,0 +1,13 @@ +registry: + # Plugin registry settings example + url: 'https://github.com/{name}/{repository}/releases/download/{tag}' + name: '' + repository: '' + tag: 'latest' +logging: + # Setting the log level: CRITICAL, ERROR, WARNING, INFO, DEBUG + level: 'DEBUG' +plugins: + # Packages to download from registry + - advance-plugin + - sample-plugin \ No newline at end of file diff --git a/usecase/__init__.py b/usecase/__init__.py new file mode 100644 index 0000000..b248ee5 --- /dev/null +++ b/usecase/__init__.py @@ -0,0 +1 @@ +from .interactors import PluginUseCase diff --git a/usecase/interactors.py b/usecase/interactors.py new file mode 100644 index 0000000..6bf6f73 --- /dev/null +++ b/usecase/interactors.py @@ -0,0 +1,78 @@ +import os +from importlib import import_module +from logging import Logger +from typing import List, Any, Dict + +from engine import IPluginRegistry, PluginCore +from util import LogUtil +from .utilities import PluginUtility + + +class PluginUseCase: + _logger: Logger + modules: List[type] + + def __init__(self, options: Dict) -> None: + self._logger = LogUtil.create(options['log_level']) + self.plugins_package: str = options['directory'] + self.plugin_util = PluginUtility(self._logger) + self.modules = list() + + def __check_loaded_plugin_state(self, plugin_module: Any): + if len(IPluginRegistry.plugin_registries) > 0: + latest_module = IPluginRegistry.plugin_registries[-1] + latest_module_name = latest_module.__module__ + current_module_name = plugin_module.__name__ + if current_module_name == latest_module_name: + self._logger.debug(f'Successfully imported module `{current_module_name}`') + self.modules.append(latest_module) + else: + self._logger.error( + f'Expected to import -> `{current_module_name}` but got -> `{latest_module_name}`' + ) + # clear plugins from the registry when we're done with them + IPluginRegistry.plugin_registries.clear() + else: + self._logger.error(f'No plugin found in registry for module: {plugin_module}') + + def __search_for_plugins_in(self, plugins_path: List[str], package_name: str): + for directory in plugins_path: + entry_point = self.plugin_util.setup_plugin_configuration(package_name, directory) + if entry_point is not None: + plugin_name, plugin_ext = os.path.splitext(entry_point) + # Importing the module will cause IPluginRegistry to invoke it's __init__ fun + import_target_module = f'.{directory}.{plugin_name}' + module = import_module(import_target_module, package_name) + self.__check_loaded_plugin_state(module) + else: + self._logger.debug(f'No valid plugin found in {package_name}') + + def discover_plugins(self, reload: bool): + """ + Discover the plugin classes contained in Python files, given a + list of directory names to scan. + """ + if reload: + self.modules.clear() + IPluginRegistry.plugin_registries.clear() + self._logger.debug(f'Searching for plugins under package {self.plugins_package}') + plugins_path = PluginUtility.filter_plugins_paths(self.plugins_package) + package_name = os.path.basename(os.path.normpath(self.plugins_package)) + self.__search_for_plugins_in(plugins_path, package_name) + + @staticmethod + def register_plugin(module: type, logger: Logger) -> PluginCore: + """ + Create a plugin instance from the given module + :param module: module to initialize + :param logger: logger for the module to use + :return: a high level plugin + """ + return module(logger) + + @staticmethod + def hook_plugin(plugin: PluginCore): + """ + Return a function accepting commands. + """ + return plugin.invoke diff --git a/usecase/utilities.py b/usecase/utilities.py new file mode 100644 index 0000000..de3fd36 --- /dev/null +++ b/usecase/utilities.py @@ -0,0 +1,104 @@ +import os +import subprocess +import sys + +from logging import Logger +from subprocess import CalledProcessError +from typing import List, Dict, Optional + +import pkg_resources +from dacite import from_dict, ForwardReferenceError, UnexpectedDataError, WrongTypeError, MissingValueError +from pkg_resources import Distribution + +from model import PluginConfig, DependencyModule +from util import FileSystem + + +class PluginUtility: + __IGNORE_LIST = ['__pycache__'] + + def __init__(self, logger: Logger) -> None: + super().__init__() + self._logger = logger + + @staticmethod + def __filter_unwanted_directories(name: str) -> bool: + return not PluginUtility.__IGNORE_LIST.__contains__(name) + + @staticmethod + def filter_plugins_paths(plugins_package) -> List[str]: + """ + filters out a list of unwanted directories + :param plugins_package: + :return: list of directories + """ + return list( + filter( + PluginUtility.__filter_unwanted_directories, + os.listdir(plugins_package) + ) + ) + + @staticmethod + def __get_missing_packages( + installed: List[Distribution], + required: Optional[List[DependencyModule]] + ) -> List[DependencyModule]: + missing = list() + if required is not None: + installed_packages: List[str] = [pkg.project_name for pkg in installed] + for required_pkg in required: + if not installed_packages.__contains__(required_pkg.name): + missing.append(required_pkg) + return missing + + def __manage_requirements(self, package_name: str, plugin_config: PluginConfig): + installed_packages: List[Distribution] = list( + filter(lambda pkg: isinstance(pkg, Distribution), pkg_resources.working_set) + ) + missing_packages = self.__get_missing_packages(installed_packages, plugin_config.requirements) + for missing in missing_packages: + self._logger.info(f'Preparing installation of module: {missing} for package: {package_name}') + try: + python = sys.executable + exit_code = subprocess.check_call( + [python, '-m', 'pip', 'install', missing.__str__()], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + self._logger.info( + f'Installation of module: {missing} for package: {package_name} was returned exit code: {exit_code}' + ) + except CalledProcessError as e: + self._logger.error(f'Unable to install package {missing}', e) + + def __read_configuration(self, module_path) -> Optional[PluginConfig]: + try: + plugin_config_data = FileSystem.load_configuration('plugin.yaml', module_path) + plugin_config = from_dict(data_class=PluginConfig, data=plugin_config_data) + return plugin_config + except FileNotFoundError as e: + self._logger.error('Unable to read configuration file', e) + except (NameError, ForwardReferenceError, UnexpectedDataError, WrongTypeError, MissingValueError) as e: + self._logger.error('Unable to parse plugin configuration to data class', e) + return None + + def setup_plugin_configuration(self, package_name, module_name) -> Optional[str]: + """ + Handles primary configuration for a give package and module + :param package_name: package of the potential plugin + :param module_name: module of the potential plugin + :return: a module name to import + """ + # if the item has not folder we will assume that it is a directory + module_path = os.path.join(FileSystem.get_plugins_directory(), module_name) + if os.path.isdir(module_path): + self._logger.debug(f'Checking if configuration file exists for module: {module_name}') + plugin_config: Optional[PluginConfig] = self.__read_configuration(module_path) + if plugin_config is not None: + self.__manage_requirements(package_name, plugin_config) + return plugin_config.runtime.main + else: + self._logger.debug(f'No configuration file exists for module: {module_name}') + self._logger.debug(f'Module: {module_name} is not a directory, skipping scanning phase') + return None diff --git a/util/__init__.py b/util/__init__.py new file mode 100644 index 0000000..21ebb76 --- /dev/null +++ b/util/__init__.py @@ -0,0 +1 @@ +from .helpers import FileSystem, LogUtil diff --git a/util/helpers.py b/util/helpers.py new file mode 100644 index 0000000..23d3ca2 --- /dev/null +++ b/util/helpers.py @@ -0,0 +1,62 @@ +import logging +import os +import sys +from logging import Logger, StreamHandler, DEBUG +from typing import Union, Optional + +import yaml + + +class FileSystem: + + @staticmethod + def __get_base_dir(): + """At most all application packages are just one level deep""" + current_path = os.path.abspath(os.path.dirname(__file__)) + return os.path.join(current_path, '..') + + @staticmethod + def __get_config_directory() -> str: + base_dir = FileSystem.__get_base_dir() + return os.path.join(base_dir, 'settings') + + @staticmethod + def get_plugins_directory() -> str: + base_dir = FileSystem.__get_base_dir() + return os.path.join(base_dir, 'plugins') + + @staticmethod + def load_configuration(name: str = 'configuration.yaml', config_directory: Optional[str] = None) -> dict: + if config_directory is None: + config_directory = FileSystem.__get_config_directory() + with open(os.path.join(config_directory, name)) as file: + input_data = yaml.safe_load(file) + return input_data + + +class LogUtil(Logger): + __FORMATTER = "%(asctime)s — %(name)s — %(levelname)s — %(funcName)s:%(lineno)d — %(message)s" + + def __init__( + self, + name: str, + log_format: str = __FORMATTER, + level: Union[int, str] = DEBUG, + *args, + **kwargs + ) -> None: + super().__init__(name, level) + self.formatter = logging.Formatter(log_format) + self.addHandler(self.__get_stream_handler()) + + def __get_stream_handler(self) -> StreamHandler: + handler = StreamHandler(sys.stdout) + handler.setFormatter(self.formatter) + return handler + + @staticmethod + def create(log_level: str = 'DEBUG') -> Logger: + logging.setLoggerClass(LogUtil) + logger = logging.getLogger('plugin.architecture') + logger.setLevel(log_level) + return logger