Update Jenkinsfile

remove unused RUN_xx.launch script

Dockerfile

@@ -6,13 +6,9 @@
 # You should have received a copy of the license along with this
 # work.  If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
 
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt update
-RUN apt install -y python3.9 python3-virtualenv python3-pip git python3.9-venv weasyprint
-
-RUN python3 -m pip install --upgrade pip
-RUN python3 -m pip install --upgrade virtualenv
-RUN python3 -m pip install --upgrade setuptools wheel build
+RUN apt install -y python3.11 python3-virtualenv python3-pip git python3-venv weasyprint

Jenkinsfile

@@ -184,7 +184,7 @@ pipeline {
                 sh("virtualenv --pip=embed --setuptools=embed --wheel=embed --no-periodic-update --activators bash,python TOOLS_ENV")
 				
 				sh(". ~/BUILD_ENV/bin/activate && pip install --upgrade setuptools build pip")
-                sh(". ~/BUILD_ENV/bin/activate && pip install --upgrade 'copier==8.*' jinja2-slug toml")
+                sh(". ~/BUILD_ENV/bin/activate && pip install --upgrade 'copier==9.*' jinja2-slug toml")
 				
                 sh(". ~/TOOLS_ENV/bin/activate && pip install simple_rest_client requests twine packaging")
                 
@@ -424,8 +424,16 @@ pipeline {
             post {
                 always {      
                     dir("gitrepo") {       
-                        publishCoverage adapters: [cobertura(mergeToOneReport: true, path: "helpers-results/types_check/cobertura.xml")]
-                        junit 'helpers-results/types_check/junit.xml'           
+                        //publish coverage
+                        recordCoverage( sourceDirectories: [[path: 'src']],
+                                        tools: [[parser: 'COBERTURA', pattern: 'helpers-results/types_check/cobertura.xml']],
+                                        id: 'COBERTURA', name: 'COBERTURA Coverage',
+                                        sourceCodeRetention: 'EVERY_BUILD',)
+                        
+                        //add type check to junit result set
+                        junit 'helpers-results/types_check/junit.xml'    
+                        
+                        //publish html reports files       
                         publishHTML([
                                     reportDir:              "helpers-results/quality_check",
                                     reportFiles:            "report.html",
@@ -535,7 +543,7 @@ pipeline {
                     dir("gitrepo") {
     				    junit 'helpers-results/unit_test/*.xml'
     				    // using cobertura format (= coverage xml format)
-    				    publishCoverage adapters: [cobertura(mergeToOneReport: true, path: "helpers-results/unit_test_coverage/test_coverage.xml")]
+    				    recordCoverage(tools: [[parser: 'COBERTURA', pattern: 'helpers-results/cl_unit_test_coverage/test_coverage.xml']])
     				    publishHTML([
                                     reportDir:              "helpers-results/unit_test_coverage",
                                     reportFiles:            "index.html",

RUN_quality.launch

@@ -9,7 +9,7 @@
     </listAttribute>
     <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${project_loc}/helpers"/>
     <stringAttribute key="org.eclipse.ui.externaltools.ATTR_OTHER_WORKING_DIRECTORY" value=""/>
-    <stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="--type-check --quality-check"/>
+    <stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="--quality-check"/>
     <stringAttribute key="org.python.pydev.debug.ATTR_INTERPRETER" value="__default"/>
     <stringAttribute key="org.python.pydev.debug.ATTR_PROJECT" value="chacha_cicd_helper"/>
     <stringAttribute key="process_factory_id" value="org.python.pydev.debug.processfactory.PyProcessFactory"/>

RUN_types.launch

@@ -9,7 +9,7 @@
     </listAttribute>
     <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${project_loc}/helpers"/>
     <stringAttribute key="org.eclipse.ui.externaltools.ATTR_OTHER_WORKING_DIRECTORY" value=""/>
-    <stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="--changelog-gen"/>
+    <stringAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS" value="--type-check"/>
     <stringAttribute key="org.python.pydev.debug.ATTR_INTERPRETER" value="__default"/>
     <stringAttribute key="org.python.pydev.debug.ATTR_PROJECT" value="chacha_cicd_helper"/>
     <stringAttribute key="process_factory_id" value="org.python.pydev.debug.processfactory.PyProcessFactory"/>

helpers/__main__.py

@@ -10,11 +10,17 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from pathlib import Path
-import tomli
+
+b_use_tomli=False
+try:
+    import tomllib
+except ImportError:
+    import tomli
+    b_use_tomli=True
+    
 import argparse
-import os
 import logging
-import sys
+
 
 if __package__ == "helpers":
     # when calling the module from: > python -m helpers
@@ -37,7 +43,10 @@ if __name__ == "__main__":
     project_rootdir_path = Path(__file__).parent.parent.absolute()
 
     with open(project_rootdir_path / "pyproject.toml", mode="rb") as fp:
-        pyproject = tomli.load(fp)
+        if b_use_tomli:
+            pyproject = tomli.load(fp)
+        else:
+            pyproject = tomllib.load(fp)
 
     parser = argparse.ArgumentParser(
         prog="continuous-integration-helper", description="A tiny set of scripts to help continous integration on python"

pyproject.toml

@@ -31,18 +31,21 @@ classifiers = [
   "Development Status :: 4 - Beta",
   "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
     'importlib-metadata; python_version<"3.9"',
     'typed-argument-parser==1.*',
     'packaging',
+    'tomli; python_version<"3.11"',
     "junitparser>=2.8",
     "junit2html>=30.1",
     "xmlrunner>=1.7",
     "mypy>=0.99",
     "coverage>=7.0",
     "radon>=5.1",
-    "pylint>=2.15",
+    "pylint>=2.15,<3",
     "pylint-json2html>=0.4",
     "pandas>=1.5",
     "mypy[reports]>=0.99",
@@ -67,9 +70,26 @@ include-package-data = true
 where = ["src"]
 
 [tool.setuptools.package-data]
-"chacha_cicd_helper.data" = ["*.*"]
 "chacha_cicd_helper" = ["py.typed"]
 
+[[tool.mypy.overrides]]
+module = "tomli"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "tomllib"
+ignore_missing_imports = true
+
+[tool.coverage.run]
+cover_pylib = false
+branch = true
+data_file="helpers-results/cl_unit_test_raw_coverage/.coverage"
+# debug = ["config","multiproc","process"]
+parallel = true
+concurrency = [
+    'thread'
+]
+
 [project.urls]
 Homepage        = "https://chacha.ddns.net/gitea/chacha/chacha_cicd_helper"
 Documentation   = "https://chacha.ddns.net/mkdocs-web/chacha/chacha_cicd_helper/master/latest/"
@@ -79,7 +99,7 @@ Tracker         = "https://chacha.ddns.net/gitea/chacha/chacha_cicd_helper/issue
 test              = ["junitparser>=2.8","junit2html>=30.1","xmlrunner>=1.7","mypy>=0.99" ]
 coverage-check    = ["coverage>=7.0"]
 complexity-check  = ["radon>=5.1"]
-quality-check     = ["pylint>=2.15","pylint-json2html>=0.4","pandas>=1.5","types-PyYAML"]
+quality-check     = ["pylint>=2.15,<3","pylint-json2html>=0.4","pandas>=1.5","types-PyYAML"]
 type-check        = ["mypy[reports]>=0.99" ]
 doc-gen           = ["mkdocs>=1.4.0", "mkdocs-material>=8.5","mkdocs-pymdownx-material-extras", "mkdocs-localsearch>=0.9.0", "mkdocstrings[python]>=0.19", "mkdocs-with-pdf>=0.9.3","pyyaml>=6.0","pymdown-extensions>=9","mkdocs-markdownextradata-plugin","mkdocs-mermaid2-plugin"]
 

src/chacha_cicd_helper/__init__.py

@@ -35,3 +35,5 @@ try:  # pragma: no cover
 except PackageNotFoundError:  # pragma: no cover
     warnings.warn('can not read dist.metadata["Name"], assuming local test context, setting it to <chacha_cicd_helper>')
     __Name__ = "chacha_cicd_helper"
+
+from .coverage_tools import CoverageProcess

src/chacha_cicd_helper/__main__.py

@@ -16,14 +16,21 @@ import os
 import logging
 import sys
 from tap import Tap
-import tomli
 
+b_use_tomli=False
+try:
+    import tomli
+    b_use_tomli=True
+except ImportError:
+    import tomllib
+        
 from .helper_base import cl_helper_base
 from .types_check import cl_types_check
 from .quality_check import cl_quality_check
 from .unit_test import cl_unit_test
 from .doc_gen import cl_doc_gen
 from .complexity_check import cl_complexity_check
+from .install_deps import cl_install_deps
 
 
 from . import __Summuary__, __Name__
@@ -35,6 +42,7 @@ class chacha_cicd_helper_args(Tap):
     """class that describe cmd arguments"""
 
     projectpath: Union[str, None] = None
+    installdeps: bool = False
     typecheck: bool = False
     unittest: bool = False
     coveragecheck: bool = False
@@ -45,7 +53,10 @@ class chacha_cicd_helper_args(Tap):
 
     def configure(self) -> None:
         """specific arguments initializer"""
+        
         self.add_argument("-pp", "--projectpath", help="path of the python project to process", default=os.getcwd())
+        
+        self.add_argument("-id", "--installdeps", action="store_true", help="install dependencies through pip")
 
         self.add_argument("-tc", "--typecheck", action="store_true", help="enable static typing check")
 
@@ -72,39 +83,46 @@ def fct_main(i_args: list[str]) -> None:  # pylint: disable=too-complex
     args = parser.parse_args(i_args)
 
     helpers: list[type[cl_helper_base]] = []
-    if args.typecheck is True:
-        helpers.append(cl_types_check)
-
-    if args.unittest is True:
-        helpers.append(cl_unit_test)
-
-    if args.coveragecheck is True:
+    
+    if args.installdeps is True:
+            helpers.append(cl_install_deps)
+    else:
+        if args.typecheck is True:
+            helpers.append(cl_types_check)
+    
         if args.unittest is True:
-            cl_unit_test.enable_coverage_check = True
-        else:
-            raise RuntimeError("unit-test is required to enable coverage-check")
-
-    if args.qualitycheck is True:
-        helpers.append(cl_quality_check)
-
-    if args.docgen is True:
-        helpers.append(cl_doc_gen)
-
-    if args.docgenpdf is True:
+            helpers.append(cl_unit_test)
+    
+        if args.coveragecheck is True:
+            if args.unittest is True:
+                cl_unit_test.enable_coverage_check = True
+            else:
+                raise RuntimeError("unit-test is required to enable coverage-check")
+    
+        if args.qualitycheck is True:
+            helpers.append(cl_quality_check)
+    
         if args.docgen is True:
-            cl_doc_gen.enable_gen_pdf = True
-        else:
-            raise RuntimeError("doc-gen is required to enable doc-gen-pdf")
-
-    if args.complexitycheck is True:
-        helpers.append(cl_complexity_check)
+            helpers.append(cl_doc_gen)
+    
+        if args.docgenpdf is True:
+            if args.docgen is True:
+                cl_doc_gen.enable_gen_pdf = True
+            else:
+                raise RuntimeError("doc-gen is required to enable doc-gen-pdf")
+    
+        if args.complexitycheck is True:
+            helpers.append(cl_complexity_check)
 
     project_rootdir_path = Path(os.getcwd()) if args.projectpath is None else Path(args.projectpath)
 
     print(f"Working directory: {project_rootdir_path}")
 
     with open(project_rootdir_path / "pyproject.toml", mode="rb") as fp:
-        pyproject = tomli.load(fp)
+        if b_use_tomli:
+            pyproject = tomli.load(fp)
+        else:
+            pyproject = tomllib.load(fp)
 
     for helper in helpers:
         helper.set_context(project_rootdir_path, pyproject)

src/chacha_cicd_helper/coverage_tools.py

@@ -0,0 +1,16 @@
+from coverage import Coverage
+from multiprocessing import Process
+import os
+
+
+class CoverageProcess(Process):
+    def run(self):
+        cov = Coverage(config_file=True, data_suffix=os.getpid(), auto_data=True)
+        cov._warn_no_data = False
+        cov.start()
+
+        try:
+            super().run()
+        finally:
+            cov.stop()
+            cov.save()

src/chacha_cicd_helper/install_deps.py

@@ -0,0 +1,39 @@
+# chacha_cicd_helper (c) by chacha
+#
+# chacha_cicd_helper is licensed under a
+# Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International Unported License.
+#
+# You should have received a copy of the license along with this
+# work.  If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
+
+"""module that handle code documentation generation"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pprint import  pprint
+
+from .helper_base import cl_helper_base
+
+
+class cl_install_deps(cl_helper_base):
+    """dependencies installer implementation class"""
+
+
+    @classmethod
+    def do_job(cls) -> None:
+        """helper job method implementation"""
+        deps=[]
+        opt_deps={}
+        if 'project' in cls.pyproject:
+            prj = cls.pyproject["project"]
+            if 'dependencies' in prj:
+                deps = prj['dependencies']
+            if 'optional-dependencies'in prj:
+                opt_deps = prj['optional-dependencies']
+        if len(deps)>0:
+            subprocess.check_call([sys.executable, "-m", "pip", "install", "--upgrade"] + deps)
+        for opt_key in opt_deps.keys():
+            if len(opt_deps[opt_key]) > 0:
+                subprocess.check_call([sys.executable, "-m", "pip", "install", "--upgrade"] + opt_deps[opt_key])

src/chacha_cicd_helper/unit_test.py

@@ -45,7 +45,7 @@ class cl_unit_test(cl_helper_withresults_base):
             # we start coverage now because module files discovery is part of the coverage measurement
             CoverageReportPath = Path(str(cls.get_result_dir()) + "_coverage")
             cls._reset_dir(CoverageReportPath)
-            cov = coverage.Coverage(cover_pylib=False, branch=True, source_pkgs=["src." + cls.pyproject["project"]["name"]])
+            cov = coverage.Coverage(config_file=True, source_pkgs=["src." + cls.pyproject["project"]["name"]])
             cov.start()
 
         package_tests = test_loader.discover(
@@ -63,6 +63,7 @@ class cl_unit_test(cl_helper_withresults_base):
         if cls.enable_coverage_check is True:
             cov.stop()
             cov.save()
+            cov.combine()
             cov.html_report(directory=str(CoverageReportPath))
             cov.xml_report(outfile=str(CoverageReportPath / f"{cls.CoverageReportName}.xml"))
 

test/test_main.py

@@ -39,7 +39,13 @@ class Test_main(unittest.TestCase):
     @unittest.skip
     def test_help_print(self):
         with redirect_stdout(StringIO()) as capted_stdout, redirect_stderr(StringIO()) as capted_stderr:
-            fct_main(["--typecheck"])
-            # fct_main(["-h", "-pp", str(testdir_path.parent.resolve())])
+            fct_main(["-h", "-pp", str(testdir_path.parent.resolve())])
+        print(capted_stdout.getvalue())
+        print(capted_stderr.getvalue())
+
+
+    def test_install_deps(self):
+        with redirect_stdout(StringIO()) as capted_stdout, redirect_stderr(StringIO()) as capted_stderr:
+            fct_main(["--installdeps", "-pp", str(testdir_path.parent.resolve())])
         print(capted_stdout.getvalue())
         print(capted_stderr.getvalue())