Merge branch 'develop'

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (plugin-architecture)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/plugin-architecture.iml" filepath="$PROJECT_DIR$/.idea/plugin-architecture.iml" />
+    </modules>
+  </component>
+</project>

.idea/plugin-architecture.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.8 (plugin-architecture)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

README.md

@@ -1,25 +1,23 @@
 # plugin-architecture
 
-## Fundamental plugin concepts
+__Sample plugin architecture for python__
+
+### Fundamental plugin concepts
 
 - Discovery
 - Registration
-- Application hooks to which plugins attach (aka. "mount points")
-- Exposing application capabilities back to plugins (aka. extension API)
+- Application hooks to which plugins attach
+- Exposing application capabilities back to plugins
 
 #### Discovery
 
 This is the mechanism by which a running application can find out which plugins it has at its disposal. 
-To "discover" a plugin, one has to look in certain places, and also know what to look for. In our example, 
-the discover_plugins function implements this - plugins are Python classes that inherit from a known base class, 
-contained in modules located in known places.
+To "discover" a plugin, one has to look in certain places, and also know what to look for.
 
 #### Registration
 
 This is the mechanism by which a plugin tells an application - "I'm here, ready to do work". 
-Admittedly, registration usually has a large overlap with discovery, 
-but I still want to keep the two concepts separate since it makes things more explicit 
-(not in all languages registration is as automatic as our example demonstrates).
+Admittedly, registration usually has a large overlap with discovery.
 
 #### Application hooks
 
@@ -27,15 +25,10 @@ Hooks are also called "mount points" or "extension points".
 These are the places where the plugin can "attach" itself to the application, 
 signaling that it wants to know about certain events and participate in the flow. 
 The exact nature of hooks is very much dependent on the application. 
-In our example, hooks allow plugins to intervene in the text-to-HTML transformation process performed by the application. 
-The example also demonstrates both coarse grained hooks (processing the whole contents) and fine grained hooks 
-(processing only certain marked-up chunks).
 
 #### Exposing application API to plugins
 
 To make plugins truly powerful and versatile, the application needs to give them access to itself, 
-by means of exposing an API the plugins can use. 
-In our example the API is relatively simple - the application simply passes some of its own internal objects to the plugins. 
-APIs tend to get much more complex when multiple languages are involved. I hope to show some interesting examples in future articles.
+by means of exposing an API the plugins can use.
 
-**[Read full blog here](https://eli.thegreenplace.net/2012/08/07/fundamental-concepts-of-plugin-infrastructures)**
+Inspired by: **[fundamental-concepts-of-plugin-infrastructures](https://eli.thegreenplace.net/2012/08/07/fundamental-concepts-of-plugin-infrastructures)**

app.py

@@ -0,0 +1,46 @@
+import argparse
+
+from engine import PluginEngine
+from util import FileSystem
+
+
+def __description() -> str:
+    return "Create your own anime meta data"
+
+
+def __usage() -> str:
+    return "vrv-meta.py --service vrv"
+
+
+def __init_cli() -> argparse:
+    parser = argparse.ArgumentParser(description=__description(), usage=__usage())
+    parser.add_argument(
+        '-l', '--log', default='DEBUG', help="""
+        Specify log level which should use. Default will always be DEBUG, choose between the following options
+        CRITICAL, ERROR, WARNING, INFO, DEBUG
+        """
+    )
+    parser.add_argument(
+        '-d', '--directory', default=f'{FileSystem.get_plugins_directory()}', help="""
+        (Optional) Supply a directory where plugins should be loaded from. The default is ./plugins
+        """
+    )
+    return parser
+
+
+def __print_program_end() -> None:
+    print("-----------------------------------")
+    print("End of execution")
+    print("-----------------------------------")
+
+
+def __init_app(parameters: dict) -> None:
+    PluginEngine(options=parameters).start()
+
+
+if __name__ == '__main__':
+    __cli_args = __init_cli().parse_args()
+    __init_app({
+        'log_level': __cli_args.log,
+        'directory': __cli_args.directory
+    })

engine/__init__.py

@@ -0,0 +1,2 @@
+from .engine_contract import PluginCore, IPluginRegistry
+from .engine_core import PluginEngine

engine/engine_contract.py

@@ -0,0 +1,36 @@
+from logging import Logger
+from typing import Optional, List
+
+from model import Meta, Device
+
+
+class IPluginRegistry(type):
+    plugin_registries: List[type] = list()
+
+    def __init__(cls, name, bases, attrs):
+        super().__init__(cls)
+        if name != 'PluginCore':
+            IPluginRegistry.plugin_registries.append(cls)
+
+
+class PluginCore(object, metaclass=IPluginRegistry):
+    """
+    Plugin core class
+    """
+
+    meta: Optional[Meta]
+
+    def __init__(self, logger: Logger) -> None:
+        """
+        Entry init block for plugins
+        :param logger: logger that plugins can make use of
+        """
+        self._logger = logger
+
+    def invoke(self, **args) -> Device:
+        """
+        Starts main plugin flow
+        :param args: possible arguments for the plugin
+        :return: a device for the plugin
+        """
+        pass

engine/engine_core.py

@@ -0,0 +1,31 @@
+from logging import Logger
+
+from usecase import PluginUseCase
+from util import LogUtil
+
+
+class PluginEngine:
+    _logger: Logger
+
+    def __init__(self, **args) -> None:
+        self._logger = LogUtil.create(args['options']['log_level'])
+        self.use_case = PluginUseCase(args['options'])
+
+    def start(self) -> None:
+        self.__reload_plugins()
+        self.__invoke_on_plugins('Q')
+
+    def __reload_plugins(self) -> None:
+        """Reset the list of all plugins and initiate the walk over the main
+        provided plugin package to load all available plugins
+        """
+        self.use_case.discover_plugins(True)
+
+    def __invoke_on_plugins(self, command: chr):
+        """Apply all of the plugins on the argument supplied to this function
+        """
+        for module in self.use_case.modules:
+            plugin = self.use_case.register_plugin(module, self._logger)
+            delegate = self.use_case.hook_plugin(plugin)
+            device = delegate(command=command)
+            self._logger.info(f'Loaded device: {device}')

logs/README.md

@@ -0,0 +1 @@
+Placeholder for filestream logs, which we are currently not using

model/__init__.py

@@ -0,0 +1 @@
+from .models import Meta, Device, PluginConfig, DependencyModule, PluginRunTimeOption

model/models.py

@@ -0,0 +1,47 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+@dataclass
+class PluginRunTimeOption(object):
+    main: str
+    tests: Optional[List[str]]
+
+
+@dataclass
+class DependencyModule:
+    name: str
+    version: str
+
+    def __str__(self) -> str:
+        return f'{self.name}=={self.version}'
+
+
+@dataclass
+class PluginConfig:
+    name: str
+    alias: str
+    creator: str
+    runtime: PluginRunTimeOption
+    repository: str
+    description: str
+    version: str
+    requirements: Optional[List[DependencyModule]]
+
+
+@dataclass
+class Meta:
+    name: str
+    description: str
+    version: str
+
+    def __str__(self) -> str:
+        return f'{self.name}: {self.version}'
+
+
+@dataclass
+class Device:
+    name: str
+    firmware: int
+    protocol: str
+    errors: List[int]

plugins/advanced-plugin/main.py

@@ -0,0 +1,57 @@
+from logging import Logger
+from random import randint
+from time import sleep
+from typing import Optional
+
+from engine import PluginCore
+from model import Meta, Device
+
+
+class AdvanceSamplePlugin(PluginCore):
+
+    def __init__(self, logger: Logger) -> None:
+        super().__init__(logger)
+        self.meta = Meta(
+            name='Advanced Sample Plugin',
+            description='Advanced Sample plugin template',
+            version='0.0.1'
+        )
+
+    @staticmethod
+    def __simulate_operation() -> None:
+        sleep_duration = randint(1, 100) / 100
+        sleep(sleep_duration)
+
+    def __get_firmware(self) -> int:
+        self._logger.debug('Enquiring device firmware')
+        self.__simulate_operation()
+        return 0xf41c3e
+
+    def __get_protocol(self) -> str:
+        self._logger.debug('Enquiring messaging protocol')
+        self.__simulate_operation()
+        return "ASCII"
+
+    def __get_errors(self) -> [int]:
+        self._logger.debug('Enquiring device errors')
+        self.__simulate_operation()
+        return [0x2f3a6c, 0xa8e1f5]
+
+    def __create_device(self) -> Device:
+        firmware = self.__get_firmware()
+        protocol = self.__get_protocol()
+        errors = self.__get_errors()
+
+        return Device(
+            name='Advanced Sample Device',
+            firmware=firmware,
+            protocol=protocol,
+            errors=errors
+        )
+
+    def invoke(self, command: chr, protocol: Optional[str] = None) -> Device:
+        self._logger.debug(f'Command: {command} -> {self.meta}')
+        device = self.__create_device()
+        if device.protocol != protocol:
+            self._logger.warning(f'Device does not support protocol supplied protocol')
+        return device

plugins/advanced-plugin/plugin.yaml

@@ -0,0 +1,15 @@
+name: 'Advanced Plugin'
+alias: 'advanced-plugin'
+creator: 'wax911'
+runtime:
+  main: 'main.py'
+  tests:
+    - 'tests/core.py'
+repository: 'https://github.com/wax911/advanced-plugin'
+description: 'Advanced sample-plugin plugin template'
+version: '0.0.1'
+requirements:
+  - name: 'PyYAML'
+    version: '5.3.1'
+  - name: 'pytz'
+    version: '2019.3'

plugins/advanced-plugin/test/core.py

@@ -0,0 +1,10 @@
+import unittest
+
+
+class MyTestCase(unittest.TestCase):
+    def test_something(self):
+        self.assertEqual(True, False)
+
+
+if __name__ == '__main__':
+    unittest.main()

plugins/sample-plugin/plugin.yaml

@@ -0,0 +1,15 @@
+name: 'Sample Plugin'
+alias: 'sample-plugin'
+creator: 'wax911'
+runtime:
+  main: 'sample.py'
+  tests:
+    - 'tests/core.py'
+repository: 'https://github.com/wax911/sample-plugin'
+description: 'Sample plugin template'
+version: '0.0.2'
+requirements:
+  - name: 'PyYAML'
+    version: '5.3.1'
+  - name: 'pytz'
+    version: '2019.3'

plugins/sample-plugin/sample.py

@@ -0,0 +1,29 @@
+from logging import Logger
+
+from engine import PluginCore
+from model import Meta, Device
+
+
+class SamplePlugin(PluginCore):
+
+    def __init__(self, logger: Logger) -> None:
+        super().__init__(logger)
+        self.meta = Meta(
+            name='Sample Plugin',
+            description='Sample plugin template',
+            version='0.0.1'
+        )
+
+    @staticmethod
+    def __create_device() -> Device:
+        return Device(
+            name='Sample Device',
+            firmware=0xa2c3f,
+            protocol='SAMPLE',
+            errors=[0x0000]
+        )
+
+    def invoke(self, command: chr) -> Device:
+        self._logger.debug(f'Command: {command} -> {self.meta}')
+        device = self.__create_device()
+        return device

plugins/sample-plugin/test/core.py

@@ -0,0 +1,10 @@
+import unittest
+
+
+class MyTestCase(unittest.TestCase):
+    def test_something(self):
+        self.assertEqual(True, False)
+
+
+if __name__ == '__main__':
+    unittest.main()

requirements.txt

@@ -0,0 +1,3 @@
+PyYAML==5.3.1
+packaging==20.4
+dacite==1.5.0

settings/configuration.yaml

@@ -0,0 +1,13 @@
+registry:
+  # Plugin registry settings example
+  url: 'https://github.com/{name}/{repository}/releases/download/{tag}'
+  name: ''
+  repository: ''
+  tag: 'latest'
+logging:
+  # Setting the log level: CRITICAL, ERROR, WARNING, INFO, DEBUG
+  level: 'DEBUG'
+plugins:
+  # Packages to download from registry
+  - advance-plugin
+  - sample-plugin

usecase/__init__.py

@@ -0,0 +1 @@
+from .interactors import PluginUseCase

usecase/interactors.py

@@ -0,0 +1,78 @@
+import os
+from importlib import import_module
+from logging import Logger
+from typing import List, Any, Dict
+
+from engine import IPluginRegistry, PluginCore
+from util import LogUtil
+from .utilities import PluginUtility
+
+
+class PluginUseCase:
+    _logger: Logger
+    modules: List[type]
+
+    def __init__(self, options: Dict) -> None:
+        self._logger = LogUtil.create(options['log_level'])
+        self.plugins_package: str = options['directory']
+        self.plugin_util = PluginUtility(self._logger)
+        self.modules = list()
+
+    def __check_loaded_plugin_state(self, plugin_module: Any):
+        if len(IPluginRegistry.plugin_registries) > 0:
+            latest_module = IPluginRegistry.plugin_registries[-1]
+            latest_module_name = latest_module.__module__
+            current_module_name = plugin_module.__name__
+            if current_module_name == latest_module_name:
+                self._logger.debug(f'Successfully imported module `{current_module_name}`')
+                self.modules.append(latest_module)
+            else:
+                self._logger.error(
+                    f'Expected to import -> `{current_module_name}` but got -> `{latest_module_name}`'
+                )
+            # clear plugins from the registry when we're done with them
+            IPluginRegistry.plugin_registries.clear()
+        else:
+            self._logger.error(f'No plugin found in registry for module: {plugin_module}')
+
+    def __search_for_plugins_in(self, plugins_path: List[str], package_name: str):
+        for directory in plugins_path:
+            entry_point = self.plugin_util.setup_plugin_configuration(package_name, directory)
+            if entry_point is not None:
+                plugin_name, plugin_ext = os.path.splitext(entry_point)
+                # Importing the module will cause IPluginRegistry to invoke it's __init__ fun
+                import_target_module = f'.{directory}.{plugin_name}'
+                module = import_module(import_target_module, package_name)
+                self.__check_loaded_plugin_state(module)
+            else:
+                self._logger.debug(f'No valid plugin found in {package_name}')
+
+    def discover_plugins(self, reload: bool):
+        """
+        Discover the plugin classes contained in Python files, given a
+        list of directory names to scan.
+        """
+        if reload:
+            self.modules.clear()
+            IPluginRegistry.plugin_registries.clear()
+            self._logger.debug(f'Searching for plugins under package {self.plugins_package}')
+            plugins_path = PluginUtility.filter_plugins_paths(self.plugins_package)
+            package_name = os.path.basename(os.path.normpath(self.plugins_package))
+            self.__search_for_plugins_in(plugins_path, package_name)
+
+    @staticmethod
+    def register_plugin(module: type, logger: Logger) -> PluginCore:
+        """
+        Create a plugin instance from the given module
+        :param module: module to initialize
+        :param logger: logger for the module to use
+        :return: a high level plugin
+        """
+        return module(logger)
+
+    @staticmethod
+    def hook_plugin(plugin: PluginCore):
+        """
+        Return a function accepting commands.
+        """
+        return plugin.invoke

usecase/utilities.py

@@ -0,0 +1,104 @@
+import os
+import subprocess
+import sys
+
+from logging import Logger
+from subprocess import CalledProcessError
+from typing import List, Dict, Optional
+
+import pkg_resources
+from dacite import from_dict, ForwardReferenceError, UnexpectedDataError, WrongTypeError, MissingValueError
+from pkg_resources import Distribution
+
+from model import PluginConfig, DependencyModule
+from util import FileSystem
+
+
+class PluginUtility:
+    __IGNORE_LIST = ['__pycache__']
+
+    def __init__(self, logger: Logger) -> None:
+        super().__init__()
+        self._logger = logger
+
+    @staticmethod
+    def __filter_unwanted_directories(name: str) -> bool:
+        return not PluginUtility.__IGNORE_LIST.__contains__(name)
+
+    @staticmethod
+    def filter_plugins_paths(plugins_package) -> List[str]:
+        """
+        filters out a list of unwanted directories
+        :param plugins_package:
+        :return: list of directories
+        """
+        return list(
+            filter(
+                PluginUtility.__filter_unwanted_directories,
+                os.listdir(plugins_package)
+            )
+        )
+
+    @staticmethod
+    def __get_missing_packages(
+            installed: List[Distribution],
+            required: Optional[List[DependencyModule]]
+    ) -> List[DependencyModule]:
+        missing = list()
+        if required is not None:
+            installed_packages: List[str] = [pkg.project_name for pkg in installed]
+            for required_pkg in required:
+                if not installed_packages.__contains__(required_pkg.name):
+                    missing.append(required_pkg)
+        return missing
+
+    def __manage_requirements(self, package_name: str, plugin_config: PluginConfig):
+        installed_packages: List[Distribution] = list(
+            filter(lambda pkg: isinstance(pkg, Distribution), pkg_resources.working_set)
+        )
+        missing_packages = self.__get_missing_packages(installed_packages, plugin_config.requirements)
+        for missing in missing_packages:
+            self._logger.info(f'Preparing installation of module: {missing} for package: {package_name}')
+            try:
+                python = sys.executable
+                exit_code = subprocess.check_call(
+                    [python, '-m', 'pip', 'install', missing.__str__()],
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL
+                )
+                self._logger.info(
+                    f'Installation of module: {missing} for package: {package_name} was returned exit code: {exit_code}'
+                )
+            except CalledProcessError as e:
+                self._logger.error(f'Unable to install package {missing}', e)
+
+    def __read_configuration(self, module_path) -> Optional[PluginConfig]:
+        try:
+            plugin_config_data = FileSystem.load_configuration('plugin.yaml', module_path)
+            plugin_config = from_dict(data_class=PluginConfig, data=plugin_config_data)
+            return plugin_config
+        except FileNotFoundError as e:
+            self._logger.error('Unable to read configuration file', e)
+        except (NameError, ForwardReferenceError, UnexpectedDataError, WrongTypeError, MissingValueError) as e:
+            self._logger.error('Unable to parse plugin configuration to data class', e)
+        return None
+
+    def setup_plugin_configuration(self, package_name, module_name) -> Optional[str]:
+        """
+        Handles primary configuration for a give package and module
+        :param package_name: package of the potential plugin
+        :param module_name: module of the potential plugin
+        :return: a module name to import
+        """
+        # if the item has not folder we will assume that it is a directory
+        module_path = os.path.join(FileSystem.get_plugins_directory(), module_name)
+        if os.path.isdir(module_path):
+            self._logger.debug(f'Checking if configuration file exists for module: {module_name}')
+            plugin_config: Optional[PluginConfig] = self.__read_configuration(module_path)
+            if plugin_config is not None:
+                self.__manage_requirements(package_name, plugin_config)
+                return plugin_config.runtime.main
+            else:
+                self._logger.debug(f'No configuration file exists for module: {module_name}')
+        self._logger.debug(f'Module: {module_name} is not a directory, skipping scanning phase')
+        return None

util/__init__.py

@@ -0,0 +1 @@
+from .helpers import FileSystem, LogUtil

util/helpers.py

@@ -0,0 +1,62 @@
+import logging
+import os
+import sys
+from logging import Logger, StreamHandler, DEBUG
+from typing import Union, Optional
+
+import yaml
+
+
+class FileSystem:
+
+    @staticmethod
+    def __get_base_dir():
+        """At most all application packages are just one level deep"""
+        current_path = os.path.abspath(os.path.dirname(__file__))
+        return os.path.join(current_path, '..')
+
+    @staticmethod
+    def __get_config_directory() -> str:
+        base_dir = FileSystem.__get_base_dir()
+        return os.path.join(base_dir, 'settings')
+
+    @staticmethod
+    def get_plugins_directory() -> str:
+        base_dir = FileSystem.__get_base_dir()
+        return os.path.join(base_dir, 'plugins')
+
+    @staticmethod
+    def load_configuration(name: str = 'configuration.yaml', config_directory: Optional[str] = None) -> dict:
+        if config_directory is None:
+            config_directory = FileSystem.__get_config_directory()
+        with open(os.path.join(config_directory, name)) as file:
+            input_data = yaml.safe_load(file)
+        return input_data
+
+
+class LogUtil(Logger):
+    __FORMATTER = "%(asctime)s — %(name)s — %(levelname)s — %(funcName)s:%(lineno)d — %(message)s"
+
+    def __init__(
+            self,
+            name: str,
+            log_format: str = __FORMATTER,
+            level: Union[int, str] = DEBUG,
+            *args,
+            **kwargs
+    ) -> None:
+        super().__init__(name, level)
+        self.formatter = logging.Formatter(log_format)
+        self.addHandler(self.__get_stream_handler())
+
+    def __get_stream_handler(self) -> StreamHandler:
+        handler = StreamHandler(sys.stdout)
+        handler.setFormatter(self.formatter)
+        return handler
+
+    @staticmethod
+    def create(log_level: str = 'DEBUG') -> Logger:
+        logging.setLoggerClass(LogUtil)
+        logger = logging.getLogger('plugin.architecture')
+        logger.setLevel(log_level)
+        return logger