chacha/chacha_cicd_helper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 49 Wiki Activity

chore: fix all quality issues

Browse Source
This commit is contained in:
cclecle
2023-09-27 22:32:47 +01:00
parent 3363792810
commit 6a7be051bb
11 changed files with 126 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
helpers/quality_check.py
View File

@@ -64,6 +64,7 @@ class quality_check(helper_withresults_base):
"--load-plugins=pylint.extensions.mccabe",
"--output-format=json,parseable",
"--disable=invalid-name,too-few-public-methods,too-many-arguments", # ignore
"--extension-pkg-whitelist=mypy",
"--ignore=_version.py",
"--reports=y",
"--score=yes",

2
pyproject.toml
View File

@@ -79,7 +79,7 @@ Tracker = "https://chacha.ddns.net/gitea/chacha/chacha_cicd_helper/issue
test = ["junitparser>=2.8","junit2html>=30.1","xmlrunner>=1.7","mypy>=0.99" ]
coverage-check = ["coverage>=7.0"]
complexity-check = ["radon>=5.1"]
quality-check = ["pylint>=2.15","pylint-json2html>=0.4","pandas>=1.5"]
quality-check = ["pylint>=2.15","pylint-json2html>=0.4","pandas>=1.5","types-PyYAML"]
type-check = ["mypy[reports]>=0.99" ]
doc-gen = ["mkdocs>=1.4.0", "mkdocs-material>=8.5","mkdocs-pymdownx-material-extras", "mkdocs-localsearch>=0.9.0", "mkdocstrings[python]>=0.19", "mkdocs-with-pdf>=0.9.3","pyyaml>=6.0","pymdown-extensions>=9","mkdocs-markdownextradata-plugin","mkdocs-mermaid2-plugin"]

12
src/chacha_cicd_helper/__init__.py
View File

@@ -9,9 +9,8 @@
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
# pylint: disable=wrong-import-position
"""
Main module __init__ file.
"""
"""Main module __init__ file."""
from importlib.metadata import distribution, version, PackageNotFoundError
import warnings
@@ -36,10 +35,3 @@ try: # pragma: no cover
except PackageNotFoundError: # pragma: no cover
warnings.warn('can not read dist.metadata["Name"], assuming local test context, setting it to <chacha_cicd_helper>')
__Name__ = "chacha_cicd_helper"
from .types_check import types_check
from .quality_check import quality_check
from .unit_test import unit_test
from .doc_gen import doc_gen
from .complexity_check import complexity_check
from .__main__ import fct_main

31
src/chacha_cicd_helper/__main__.py
View File

@@ -5,7 +5,9 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""Main module"""
from __future__ import annotations
from pathlib import Path
@@ -15,11 +17,12 @@ import sys
from tap import Tap
import tomli
from . import types_check
from . import quality_check
from . import unit_test
from . import doc_gen
from . import complexity_check
from .helper_base import cl_helper_base
from .types_check import cl_types_check
from .quality_check import cl_quality_check
from .unit_test import cl_unit_test
from .doc_gen import cl_doc_gen
from .complexity_check import cl_complexity_check
from . import __Summuary__, __Name__
@@ -64,39 +67,39 @@ class chacha_cicd_helper_args(Tap):
self.add_argument("-cpc", "--complexity-check", dest="complexitycheck", action="store_true", help="enable complexity check")
def fct_main(i_args: list[str]) -> None:
def fct_main(i_args: list[str]) -> None: # pylint: disable=too-complex
"""argument processing function"""
parser = chacha_cicd_helper_args(prog=__Name__, description=__Summuary__)
args = parser.parse_args(i_args)
helpers = []
helpers: list[type[cl_helper_base]] = []
if args.typecheck is True:
helpers.append(types_check)
helpers.append(cl_types_check)
if args.unittest is True:
helpers.append(unit_test)
helpers.append(cl_unit_test)
if args.coveragecheck is True:
if args.unittest is True:
unit_test.enable_coverage_check = True
cl_unit_test.enable_coverage_check = True
else:
raise RuntimeError("unit-test is required to enable coverage-check")
if args.qualitycheck is True:
helpers.append(quality_check)
helpers.append(cl_quality_check)
if args.docgen is True:
helpers.append(doc_gen)
helpers.append(cl_doc_gen)
if args.docgenpdf is True:
if args.docgen is True:
doc_gen.enable_gen_pdf = True
cl_doc_gen.enable_gen_pdf = True
else:
raise RuntimeError("doc-gen is required to enable doc-gen-pdf")
if args.complexitycheck is True:
helpers.append(complexity_check)
helpers.append(cl_complexity_check)
project_rootdir_path = Path(args.projectpath)

21
src/chacha_cicd_helper/complexity_check.py
View File

@@ -5,27 +5,30 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""module that handle code complexity measurement"""
from __future__ import annotations
"""module that handle code complexity measurement"""
from __future__ import annotations
import statistics
import csv
from json import loads as JSON_LOADS
from radon.complexity import SCORE
from radon.cli import Config
from radon.cli.harvest import CCHarvester, HCHarvester, MIHarvester
from radon.complexity import SCORE # type: ignore
from radon.cli import Config # type: ignore
from radon.cli.harvest import CCHarvester, HCHarvester, MIHarvester # type: ignore
from .helper_base import helper_withresults_base
from .helper_base import cl_helper_withresults_base
class complexity_check(helper_withresults_base):
class cl_complexity_check(cl_helper_withresults_base):
"""complexity check implementation class"""
@classmethod
def do_job(cls):
def do_job(cls) -> None:
"""helper job method implementation"""
config = Config(
exclude="__init__\.py",
exclude=r"__init__.py", ##!!!!! . => \. ???
ignore=None,
order=SCORE,
show_closures=False,

11
src/chacha_cicd_helper/doc_gen.py
View File

@@ -5,7 +5,9 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""module that handle code documentation generation"""
from __future__ import annotations
import shutil
@@ -15,16 +17,17 @@ from distutils.dir_util import copy_tree
import yaml
from .helper_base import helper_withresults_base
from .helper_base import cl_helper_withresults_base
class doc_gen(helper_withresults_base):
class cl_doc_gen(cl_helper_withresults_base):
"""documentation generation implementation class"""
enable_gen_pdf: bool = False
@classmethod
def do_job(cls):
def do_job(cls) -> None:
"""helper job method implementation"""
# create doc root dir
doc_path = cls.project_rootdir_path / "docs"
@@ -65,7 +68,7 @@ class doc_gen(helper_withresults_base):
# little hack here, to enable / disable pdf generation using own class config
# => reason is mkdocs seems to try loading the plugin even if we disable it, so we need to
# manually process the configuration file.
with open(cls.project_rootdir_path / "mkdocs.yml", "r") as mkdocsCfgFile:
with open(cls.project_rootdir_path / "mkdocs.yml", "r", encoding="utf8") as mkdocsCfgFile:
mkdocsCfg = yaml.load(mkdocsCfgFile, Loader=yaml.Loader)
if "plugins" in mkdocsCfg:

55
src/chacha_cicd_helper/helper_base.py
View File

@@ -5,69 +5,79 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""module that describe the base helpers class"""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING: # Only imports the below statements during type checking
from typing import Union
"""module that describe the base helpers class"""
from __future__ import annotations
from abc import ABC, abstractmethod
import os
import subprocess
from pathlib import Path
import shutil
from typing import TYPE_CHECKING
class helper_base(ABC):
if TYPE_CHECKING: # Only imports the below statements during type checking
from typing import Union
class cl_helper_base(ABC):
"""helpers base class"""
project_rootdir_path: Union[Path, None] = None
pyproject: Union[dict, None] = None
current_dir: Union[Path, None] = None
project_rootdir_path: Path = Path()
pyproject: dict = {}
current_dir: Path = Path()
@classmethod
def set_context(cls, project_rootdir_path: Path, pyproject: dict):
def set_context(cls, project_rootdir_path: Path, pyproject: dict) -> None:
"""method to set contextual fields"""
cls.project_rootdir_path = project_rootdir_path
cls.pyproject = pyproject
cls.current_dir = Path(__file__).parent.absolute()
@classmethod
def get_result_dir(cls):
def get_result_dir(cls) -> Union[None, Path]:
"""retrieve the result directory path"""
return None
@staticmethod
def _create_dir(dirpath: Path):
def _create_dir(dirpath: Path) -> None:
"""helper method to create a directory"""
dirpath = Path(dirpath)
if not os.path.exists(dirpath):
os.makedirs(dirpath)
@staticmethod
def _reset_dir(dirpath: Path):
def _reset_dir(dirpath: Path) -> None:
"""helper method to reset a directory"""
dirpath = Path(dirpath)
if os.path.exists(dirpath):
shutil.rmtree(dirpath)
os.makedirs(dirpath)
@classmethod
def reset_result_dir(cls):
def reset_result_dir(cls) -> None:
"""helper method to reset the results directory"""
result_dir = cls.get_result_dir()
if result_dir is not None:
cls._reset_dir(result_dir)
@classmethod
@abstractmethod
def do_job(cls):
def do_job(cls) -> None:
"""helper job virtual method"""
raise NotImplementedError()
@classmethod
def run_cmd_(cls, cmdarray):
def run_cmd_(cls, cmdarray: list[str]):
"""helper method to run a command (piped output)"""
process = subprocess.run(cmdarray, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True)
return process.stdout
@classmethod
def run_cmd(cls, cmdarray, silent: bool = False):
"""helper method to run a command"""
p = subprocess.run(cmdarray, capture_output=True, check=True)
if not silent:
print(p.stdout.decode())
@@ -75,11 +85,14 @@ class helper_base(ABC):
return p.stdout
class helper_withresults_base(helper_base):
class cl_helper_withresults_base(cl_helper_base):
"""derived class to handle results"""
helper_results_dir: Union[Path, None] = None
@classmethod
def get_result_dir(cls):
if cls.helper_results_dir == None:
cls.helper_results_dir = cls.__name__
def get_result_dir(cls) -> Path:
"""retrieve the results directory"""
if cls.helper_results_dir is None:
cls.helper_results_dir = Path(cls.__name__)
return Path(__file__).parent.parent.absolute() / "helpers-results" / cls.helper_results_dir

54
src/chacha_cicd_helper/quality_check.py
View File

@@ -5,9 +5,11 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""module that handle code quality measurement"""
from __future__ import annotations
"""module that handle code quality measurement"""
from __future__ import annotations
from typing import TYPE_CHECKING
from contextlib import redirect_stdout, suppress
from io import StringIO
import re
@@ -17,26 +19,29 @@ import sys
import csv
import copy
import pandas
from pylint.lint import Run as pylint_Run # type: ignore
import pylint_json2html # type: ignore
import pandas # type: ignore
from pylint.lint import Run as pylint_Run
import pylint_json2html
from .helper_base import cl_helper_withresults_base
from .helper_base import helper_withresults_base
if TYPE_CHECKING: # Only imports the below statements during type checking
from typing import Any
class PyLintMetricNotFound(Warning):
pass
"""module specific warning"""
class quality_check(helper_withresults_base):
class cl_quality_check(cl_helper_withresults_base):
"""quality check implementation class"""
PylintMessageList = {}
PylintMessageList: dict[str, str] = {}
@classmethod
def GetPylintMessageList(cls):
Messagelist = {}
def GetPylintMessageList(cls) -> None:
"""helper method to parse pylint messages"""
Messagelist: dict[str, str] = {}
regex = r"^:([a-zA-Z-]+) \(([^\)]+)\)"
for line in cls.run_cmd([sys.executable, "-m", "pylint", "--list-msgs"], True).splitlines():
if res := re.search(regex, line.decode()):
@@ -44,19 +49,22 @@ class quality_check(helper_withresults_base):
cls.PylintMessageList = Messagelist
@staticmethod
def TryExtractPYReportMetric(line: str, tag: str):
regex = r"^(?:\|{0}\s*\|)(\d+)(?=\s*|)".format(tag)
def TryExtractPYReportMetric(line: str, tag: str) -> float:
"""helper method to parse pylint metrics"""
regex = rf"^(?:\|{tag}\s*\|)(\d+)(?=\s*|)"
if res := re.search(regex, line):
return float(res.group(1))
raise PyLintMetricNotFound()
@classmethod
def do_job(cls):
def do_job(cls) -> None: # pylint: disable=too-complex,too-many-locals,too-many-branches,too-many-statements
"""helper job method implementation"""
print("checking code quality ...")
cls.GetPylintMessageList()
RES_all = {}
RES_all: dict[str, Any] = {}
with StringIO() as StdOutput:
JsonContent = ""
with redirect_stdout(StdOutput):
@@ -77,6 +85,8 @@ class quality_check(helper_withresults_base):
with open(cls.get_result_dir() / "report.json", "w+", encoding="utf-8") as Outfile:
# hacky way of exctracting json + having overall score...
class TScanState(Enum):
"""Quality report parsing FSM states definition"""
TEXT_REPORT = 1
JSON_REPORT = 2
OTHER_REPORT_START = 3
@@ -181,7 +191,7 @@ class quality_check(helper_withresults_base):
if line.startswith("--------"):
ScanState = TScanState.OTHER_REPORT_END
else:
for PylintMessage in cls.PylintMessageList.keys():
for PylintMessage in cls.PylintMessageList:
with suppress(PyLintMetricNotFound):
RES_all["Messages"][PylintMessage] = cls.TryExtractPYReportMetric(line, PylintMessage)
@@ -193,7 +203,7 @@ class quality_check(helper_withresults_base):
raise RuntimeError("Invalid ScanState")
Outfile.write(JsonContent)
with open(cls.get_result_dir() / "metrics.json", "w") as json_file:
with open(cls.get_result_dir() / "metrics.json", "w", encoding="utf8") as json_file:
json.dump(RES_all, json_file)
# exporting all Data in one csv, unused atm because jenkins seems not able to select columns from csv an keep displaying all...
@@ -202,14 +212,14 @@ class quality_check(helper_withresults_base):
del RES_all_trim["Messages"]
flat_RES_all = pandas.json_normalize(RES_all_trim, sep="_").to_dict(orient="records")[0]
with open(cls.get_result_dir() / "metrics.csv", "w", newline="") as csv_file:
with open(cls.get_result_dir() / "metrics.csv", "w", newline="", encoding="utf8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=flat_RES_all.keys())
writer.writeheader()
writer.writerow(flat_RES_all)
# splited csv exports for jenkins plots: RawMetricsPercent
RES_all_percent = RES_all["RawMetricsPercent"]
with open(cls.get_result_dir() / "metrics_rawpercent.csv", "w", newline="") as csv_file:
with open(cls.get_result_dir() / "metrics_rawpercent.csv", "w", newline="", encoding="utf8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=RES_all_percent.keys())
writer.writeheader()
writer.writerow(RES_all_percent)
@@ -220,21 +230,21 @@ class quality_check(helper_withresults_base):
RES_all_stats["PersentDuplicatedLines"] = RES_all["Duplication"]["PersentDuplicatedLines"]
RES_all_stats["NbAnalysedStatments"] = RES_all["NbAnalysedStatments"]
RES_all_stats["NbAnalysedLines"] = RES_all["NbAnalysedLines"]
with open(cls.get_result_dir() / "metrics_Statistics.csv", "w", newline="") as csv_file:
with open(cls.get_result_dir() / "metrics_Statistics.csv", "w", newline="", encoding="utf8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=RES_all_stats.keys())
writer.writeheader()
writer.writerow(RES_all_stats)
# splited csv exports for jenkins plots: Statistics + Duplication
RES_all_MessagesCat = RES_all["MessagesCat"]
with open(cls.get_result_dir() / "metrics_MessagesCat.csv", "w", newline="") as csv_file:
with open(cls.get_result_dir() / "metrics_MessagesCat.csv", "w", newline="", encoding="utf8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=RES_all_MessagesCat.keys())
writer.writeheader()
writer.writerow(RES_all_MessagesCat)
# splited csv exports for jenkins plots: GlobalScore
RES_GlobalScore = {"GlobalScore": RES_all["GlobalScore"]}
with open(cls.get_result_dir() / "metrics_GlobalScore.csv", "w", newline="") as csv_file:
with open(cls.get_result_dir() / "metrics_GlobalScore.csv", "w", newline="", encoding="utf8") as csv_file:
writer = csv.DictWriter(csv_file, fieldnames=RES_GlobalScore.keys())
writer.writeheader()
writer.writerow(RES_GlobalScore)

10
src/chacha_cicd_helper/types_check.py
View File

@@ -5,21 +5,25 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""module that handle code type checking"""
from __future__ import annotations
from mypy import api
from .helper_base import helper_withresults_base
from .helper_base import cl_helper_withresults_base
class types_check(helper_withresults_base):
class cl_types_check(cl_helper_withresults_base):
"""type check implementation class"""
JUnitReportName = "junit.xml"
@classmethod
def do_job(cls):
def do_job(cls) -> None:
"""helper job method implementation"""
print("checking code typing ...")
result = api.run(
[ # project path

20
src/chacha_cicd_helper/unit_test.py
View File

@@ -5,7 +5,9 @@
#
# You should have received a copy of the license along with this
# work. If not, see <https://creativecommons.org/licenses/by-nc-sa/4.0/>.
"""module that handle code unit-test"""
from __future__ import annotations
from pathlib import Path
@@ -13,14 +15,14 @@ import os
import datetime
import unittest
import xmlrunner
from junitparser import JUnitXml
from junit2htmlreport import parser as junit2html_parser
import xmlrunner # type: ignore
from junitparser import JUnitXml # type: ignore
from junit2htmlreport import parser as junit2html_parser # type: ignore
from .helper_base import helper_withresults_base
from .helper_base import cl_helper_withresults_base
class unit_test(helper_withresults_base):
class cl_unit_test(cl_helper_withresults_base):
"""unit test implementation class"""
enable_coverage_check: bool = False
@@ -30,9 +32,11 @@ class unit_test(helper_withresults_base):
CoverageReportName: str = "test_coverage"
@classmethod
def do_job(cls):
def do_job(cls) -> None:
"""helper job method implementation"""
if cls.enable_coverage_check is True:
import coverage
import coverage # type: ignore # pylint: disable=import-outside-toplevel
# preparing unittest framework
test_loader = unittest.TestLoader()
@@ -63,7 +67,7 @@ class unit_test(helper_withresults_base):
cov.xml_report(outfile=(CoverageReportPath / f"{cls.CoverageReportName}.xml"))
# computing results (Only if xml available)
if cls.enable_full_xml_export == True:
if cls.enable_full_xml_export is True:
print("Full reports generation...")
FullReportPath = Path(str(cls.get_result_dir()) + "_full")
cls._reset_dir(FullReportPath)

2
test/test_main.py
View File

@@ -15,7 +15,7 @@ from pathlib import Path
print(__name__)
print(__package__)
from src import chacha_cicd_helper
from src.__main__ import chacha_cicd_helper
testdir_path = Path(__file__).parent.resolve()