From 1c452bc86eca9acedad209eba0a7e7232ab3cb87 Mon Sep 17 00:00:00 2001 From: MDW Date: Mon, 12 Feb 2024 02:01:51 +0100 Subject: [PATCH] Qual: Fix apstats & add phan (#28127) # Qual: Fix apstats & add phan I fixed several issues that I ran into when running apstats.php locally to test the phan integration. So this commit only proposes the updates to apstats.php. I added a test that disables phan when its configurations is not found so it should be fine to integration in develop. --- dev/tools/apstats.php | 173 +++++++++++++++++++++++++++++++++--------- 1 file changed, 137 insertions(+), 36 deletions(-) diff --git a/dev/tools/apstats.php b/dev/tools/apstats.php index 86143f0acd0..8776b4bb04f 100755 --- a/dev/tools/apstats.php +++ b/dev/tools/apstats.php @@ -2,6 +2,7 @@ + * Copyright (C) 2024 MDW * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -41,15 +42,20 @@ define('VERSION', "1.0"); $phpstanlevel = 3; // Include Dolibarr environment -require_once $path.'../../htdocs/master.inc.php'; +if (!is_readable("{$path}../../htdocs/config/config.php")) { + define("DOL_DOCUMENT_ROOT", __DIR__."/../../htdocs"); +} else { + require_once $path.'../../htdocs/master.inc.php'; +} require_once $path.'../../htdocs/core/lib/files.lib.php'; +require_once $path.'../../htdocs/core/lib/functions.lib.php'; require_once $path.'../../htdocs/core/lib/geturl.lib.php'; print '***** '.constant('PRODUCT').' - '.constant('VERSION').' *****'."\n"; if (empty($argv[1])) { - print 'You must run this tool being into the root of the project.'."\n"; - print 'Usage: '.constant('PRODUCT').'.php pathto/outputfile.html [--dir-scc=pathtoscc|disabled] [--dir-phpstan=pathtophpstan|disabled]'."\n"; - print 'Example: '.constant('PRODUCT').'.php documents/apstats/index.html --dir-scc=/snap/bin --dir-phpstan=~/git/phpstan/htdocs/includes/bin'; + print 'You must run this tool at the root of the project.'."\n"; + print 'Usage: '.constant('PRODUCT').'.php pathto/outputfile.html [--dir-scc=pathtoscc|disabled] [--dir-phpstan=pathtophpstan|disabled] [--dir-phan=path/to/phan|disabled]'."\n"; + print 'Example: '.constant('PRODUCT').'.php documents/apstats/index.html --dir-scc=/snap/bin --dir-phpstan=~/git/phpstan/htdocs/includes/bin --dir-phan=~/vendor/bin/phan'; exit(0); } @@ -64,19 +70,27 @@ if (!is_dir($outputdir)) { $dirscc = ''; $dirphpstan = ''; +$dir_phan = ''; +$datatable_script = ''; $i = 0; while ($i < $argc) { $reg = array(); - if (preg_match('/--dir-scc=(.*)$/', $argv[$i], $reg)) { + if (preg_match('/^--dir-scc=(.*)$/', $argv[$i], $reg)) { $dirscc = $reg[1]; - } - if (preg_match('/--dir-phpstan=(.*)$/', $argv[$i], $reg)) { + } elseif (preg_match('/^--dir-phpstan=(.*)$/', $argv[$i], $reg)) { $dirphpstan = $reg[1]; + } elseif (preg_match('/^--dir-phan=(.*)$/', $argv[$i], $reg)) { + $dir_phan = $reg[1]; } $i++; } +if (!is_readable("{$path}phan/config.php")) { + print "Skipping phan - configuration not found"; + // Disable phan while not integrated yet + $dir_phan = 'disabled'; +} // Start getting data @@ -86,10 +100,11 @@ $timestart = time(); $urlgit = 'https://github.com/Dolibarr/dolibarr/blob/develop/'; // Count lines of code of application +$output_arrproj = array(); +$output_arrdep = array(); if ($dirscc != 'disabled') { $commandcheck = ($dirscc ? $dirscc.'/' : '').'scc . --exclude-dir=htdocs/includes,htdocs/custom,htdocs/theme/common/fontawesome-5,htdocs/theme/common/octicons'; print 'Execute SCC to count lines of code in project: '.$commandcheck."\n"; - $output_arrproj = array(); $resexecproj = 0; exec($commandcheck, $output_arrproj, $resexecproj); @@ -97,20 +112,36 @@ if ($dirscc != 'disabled') { // Count lines of code of dependencies $commandcheck = ($dirscc ? $dirscc.'/' : '').'scc htdocs/includes htdocs/theme/common/fontawesome-5 htdocs/theme/common/octicons'; print 'Execute SCC to count lines of code in dependencies: '.$commandcheck."\n"; - $output_arrdep = array(); $resexecdep = 0; exec($commandcheck, $output_arrdep, $resexecdep); } // Get technical debt +$output_arrtd = array(); if ($dirphpstan != 'disabled') { $commandcheck = ($dirphpstan ? $dirphpstan.'/' : '').'phpstan --level='.$phpstanlevel.' -v analyze -a build/phpstan/bootstrap.php --memory-limit 5G --error-format=github'; print 'Execute PHPStan to get the technical debt: '.$commandcheck."\n"; - $output_arrtd = array(); $resexectd = 0; exec($commandcheck, $output_arrtd, $resexectd); } +$output_phan_json = array(); +$res_exec_phan = 0; +if ($dir_phan != 'disabled') { + // Get technical debt (phan) + $PHAN_CONFIG = "dev/tools/phan/config_extended.php"; + $PHAN_BASELINE = "dev/tools/phan/baseline.txt"; + $PHAN_MIN_PHP = "7.0"; + $PHAN_MEMORY_OPT = "--memory-limit 5G"; + + $commandcheck + = ($dir_phan ? $dir_phan.DIRECTORY_SEPARATOR : '') + ."phan --output-mode json $PHAN_MEMORY_OPT -k $PHAN_CONFIG -B $PHAN_BASELINE --analyze-twice --minimum-target-php-version $PHAN_MIN_PHP"; + print 'Execute Phan to get the technical debt: '.$commandcheck."\n"; + exec($commandcheck, $output_phan_json, $res_exec_phan); +} + + $arrayoflineofcode = array(); $arraycocomo = array(); $arrayofmetrics = array( @@ -173,7 +204,7 @@ foreach (array('proj', 'dep') as $source) { } // Search the max -$arrayofmax = array('Lines'=>0); +$arrayofmax = array('Lines' => 0); foreach (array('proj', 'dep') as $source) { if (!empty($arrayoflineofcode[$source])) { foreach ($arrayoflineofcode[$source] as $val) { @@ -183,8 +214,11 @@ foreach (array('proj', 'dep') as $source) { } +$nbofmonth = 2; +$delay = (3600 * 24 * 30 * $nbofmonth); + // Get stats on nb of commits -$commandcheck = "git log --shortstat --no-renames --no-merges --use-mailmap --pretty='format:%cI;%H;%aN;%aE;%ce;%s' --since='".dol_print_date(dol_now() - $delay, '%Y-%m-%d');"'"; // --since= --until=... +$commandcheck = "git log --shortstat --no-renames --no-merges --use-mailmap --pretty=".escapeshellarg('format:%cI;%H;%aN;%aE;%ce;%s')." --since=".dol_print_date(dol_now() - $delay, '%Y-%m-%d'); // --since= --until=... print 'Execute git log to get list of commits: '.$commandcheck."\n"; $output_arrglpu = array(); $resexecglpu = 0; @@ -196,14 +230,14 @@ $nbofmonth = 2; $delay = (3600 * 24 * 30 * $nbofmonth); $arrayofalerts = array(); -$commandcheck = "git log --shortstat --no-renames --no-merges --use-mailmap --pretty='format:%cI;%H;%aN;%aE;%ce;%s' --since='".dol_print_date(dol_now() - $delay, '%Y-%m-%d')."' | grep -E 'yogosha|CVE|Sec:'"; +$commandcheck = "git log --shortstat --no-renames --no-merges --use-mailmap --pretty=".escapeshellarg('format:%cI;%H;%aN;%aE;%ce;%s')." --since=".escapeshellarg(dol_print_date(dol_now() - $delay, '%Y-%m-%d'))." | grep -E ".escapeshellarg("(yogosha|CVE|Sec:)"); print 'Execute git log to get commits related to security: '.$commandcheck."\n"; $output_arrglpu = array(); $resexecglpu = 0; exec($commandcheck, $output_arrglpu, $resexecglpu); foreach ($output_arrglpu as $val) { $tmpval = cleanVal2($val); - if (preg_match('/yogosha|CVE|Sec:/i', $tmpval['title'])) { + if (preg_match('/(yogosha|CVE|Sec:)/i', $tmpval['title'])) { $alreadyfound = ''; $alreadyfoundcommitid = ''; foreach ($arrayofalerts as $val) { @@ -303,11 +337,13 @@ $html = ''."\n"; $html .= ''."\n"; $html .= ''."\n"; $html .= ''."\n"; -$html .= ''."\n"; +$html .= ''."\n"; $html .= ''."\n"; $html .= ''."\n"; $html .= ''."\n"; $html .= ''."\n"; +$html .= ''; +$html .= ''; $html .= '