QUAL: Add sqlfluff (SQL code and style check) (#29097)

* QUAL: Add sqlfluff (SQL code and style check)

# QUAL: Add sqlfluff (SQL code and style check)

This adds a validity and style check on the .sql files.
The same tool can be used to fix style (which can be set up as a
pre-commit hook).

* Ignore some sqlfluff notices

* Ignore RF04 notice, warning about the use of sql keywords

* Adjust dialects for some directories

.github/workflows/pre-commit.yml

@@ -10,6 +10,10 @@ on:
         type: string
   workflow_dispatch:
 
+concurrency:
+  group: pre-commit-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref
+    }}
+  cancel-in-progress: true
 env:
   gh_event: ${{ inputs.gh_event || github.event_name }}
 jobs:
@@ -47,12 +51,14 @@ jobs:
           cache: pip
           python-version: "3.11"
       - run: python -m pip install pre-commit
+
       # Restore previous cache of precommit
       - uses: actions/cache/restore@v4
         with:
           path: ~/.cache/pre-commit/
           key: pre-commit-4|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
-      # Run all the precommit tools (defined into pre-commit-config.yaml).
+
+      # Run all the precommit tools (defined in pre-commit-config.yaml).
       # We can force exclusion of some of them here.
       - name: Run pre-commit hooks
         env:
@@ -112,6 +118,7 @@ jobs:
           set -o pipefail
           ln -sf ~/.cache .cache  # Absolute path in .pre-commit-config.yaml
           pre-commit run --hook-stage manual -a php-cs-with-cache | tee -a ${RAW_LOG}
+          pre-commit run --hook-stage manual -a sqlfluff-lint | tee -a ${RAW_LOG}
           ls -l ~/.cache/pre-commit/
 
       - name: Convert Raw Log to Annotations
@@ -135,4 +142,3 @@ jobs:
           path: |
             ${{ env.RAW_LOG }}
             ${{ env.CS_XML }}
-          retention-days: 2

.pre-commit-config.yaml

@@ -43,6 +43,7 @@ repos:
       - id: end-of-file-fixer
       # Check that there are no completely merged file conflicts
       - id: check-merge-conflict
+        stages: [pre-commit, pre-rebase, pre-commit, pre-merge-commit]
       # Check that files with shebangs have the executable bit set (in git)
       - id: check-executables-have-shebangs
       # Check that shell files are executables
@@ -231,3 +232,26 @@ repos:
     hooks:
       - id: shellcheck
         args: [-W, "100"]
+
+  - repo: https://github.com/sqlfluff/sqlfluff
+    rev: 3.0.4
+    hooks:
+      - id: sqlfluff-lint
+        stages: [pre-commit, manual]  # manual needed for ci
+        exclude: (?x)^
+          (htdocs/includes/.*
+          |htdocs/install/doctemplates/websites/.*_template
+          |htdocs/core/menus/init_menu_auguria\.sql
+          |htdocs/install/doctemplates/websites/website_template-.*\.sql
+          |(htdocs/install/mysql/data/(llx_20_c_departements\.sql
+           |llx_accounting_account_.*\.sql)
+           |(htdocs/install/mysql/migration/3\.[256]\.0-.*\.sql)
+           )
+          |htdocs/install/mysql/migration/(1[0-5]|[456789])\.0\.0-.*\.sql
+          |htdocs/install/mysql/migration/3\.([0134789])\.0-.*\.sql
+          |htdocs/install/mysql/migration/repair\.sql
+          |htdocs/install/mysql/tables/llx_bookcal_availabilities-bookcal\.sql
+          |htdocs/install/mysql/tables/llx_categorie(_(account|actioncomm|contact|fournisseur|knowledgemanagement-knowledgemanagement|member|product|project|societe|ticket-ticket|user|warehouse|website_page-website)?\.key\.sql)
+          |htdocs/install/mysql/tables/llx_rights_def\.key\.sql
+          |htdocs/install/pgsql/functions/functions(-(don|loan|mailing|opensurvey|partnership|recruitment|website))?\.sql
+          )$

htdocs/install/pgsql/.sqlfluff

@@ -0,0 +1,2 @@
+[sqlfluff]
+dialect = postgres

htdocs/install/sqlite3/.sqlfluff

@@ -0,0 +1,2 @@
+[sqlfluff]
+dialect = sqlite

pyproject.toml

@@ -49,3 +49,22 @@ line_length = 80
 # YAMLFIX_quote_keys_and_basic_values = false
 # uote_representation = false
 # preserve_quotes = false
+
+[tool.sqlfluff.core]
+# Documentation: https://docs.sqlfluff.com/en/stable/configuration.html#default-configuration
+# (Note: the examples given are for a .ini file and need slight adaptions for pyproject.toml
+
+ignore_comment_lines = true
+sql_file_exts = ".sql"
+encoding = "utf-8"
+processes = -1
+#verbose = 1
+#exclude_rules = "LT01,CP01,RF04"
+# RF04 | Keywords should not be used as identifiers. (rowid, login, position, ...)
+exclude_rules = "LT01,LT02,LT05,LT12,LT13,CP01,CP02,CP04,CP05,RF04"
+dialect = "mysql"
+# Default byte limit is 20000, must set limit - some files are too big.
+large_file_skip_byte_limit = 100000
+
+[tool.sqlfluff.indentation]
+indent_unit = "tab"