script de netoyage des csv et securisation des variables d'environnement
This commit is contained in:
+12
-11
@@ -2,10 +2,11 @@
|
||||
"""scripts/clean_csv.py
|
||||
|
||||
Usage:
|
||||
python3 scripts/clean_csv.py [input_dir] [--out-dir OUT] [--dry-run]
|
||||
python3 scripts/clean_csv.py [input_dir] [--out-dir OUT] [--dry-run]
|
||||
|
||||
This script cleans CSV files (trim, remove 'NULL', fix column count) and
|
||||
performs simple validations (UUID-looking columns, date columns).
|
||||
Ce script nettoie les fichiers CSV (trim, suppression de 'NULL', correction du
|
||||
nombre de colonnes) et réalise des validations simples (colonnes ressemblant
|
||||
à des UUID, colonnes date).
|
||||
"""
|
||||
import csv
|
||||
import sys
|
||||
@@ -46,7 +47,7 @@ def clean_csv_file(file_path: Path, out_path: Path = None, dry_run: bool = False
|
||||
nb_cols = len(header)
|
||||
cleaned_rows.append([h.strip() for h in header])
|
||||
|
||||
# Heuristics for validations
|
||||
# Heuristiques pour les validations
|
||||
uuid_cols = [i for i, h in enumerate(header) if h.lower() == 'id' or h.lower().endswith('_id')]
|
||||
date_cols = [i for i, h in enumerate(header) if any(k in h.lower() for k in ('date', '_le', '_at'))]
|
||||
|
||||
@@ -63,15 +64,15 @@ def clean_csv_file(file_path: Path, out_path: Path = None, dry_run: bool = False
|
||||
for ci in uuid_cols:
|
||||
if ci < len(row) and row[ci]:
|
||||
if not is_uuid(row[ci]):
|
||||
errors.append(f"Line {i}: column {header[ci]} not UUID-like: {row[ci]!r}")
|
||||
errors.append(f"Ligne {i} : colonne {header[ci]} ne ressemble pas à un UUID : {row[ci]!r}")
|
||||
for ci in date_cols:
|
||||
if ci < len(row) and row[ci]:
|
||||
if not looks_like_date(row[ci]):
|
||||
errors.append(f"Line {i}: column {header[ci]} not ISO-like date: {row[ci]!r}")
|
||||
errors.append(f"Ligne {i} : colonne {header[ci]} n'est pas une date ISO-like : {row[ci]!r}")
|
||||
|
||||
cleaned_rows.append(row)
|
||||
|
||||
# Write output if not dry-run
|
||||
# Écrire la sortie si non --dry-run
|
||||
if not dry_run:
|
||||
target = out_path if out_path else file_path
|
||||
with target.open('w', encoding='utf-8', newline='') as outfile:
|
||||
@@ -99,11 +100,11 @@ def main():
|
||||
res = clean_csv_file(file_path, out_path, args.dry_run)
|
||||
results.append(res)
|
||||
|
||||
# Report
|
||||
# Rapport
|
||||
any_errors = False
|
||||
for r in results:
|
||||
if 'error' in r:
|
||||
print(f"[WARN] {r['file']}: {r['error']}")
|
||||
print(f"[AVERT] {r['file']}: {r['error']}")
|
||||
if r.get('errors'):
|
||||
any_errors = True
|
||||
print(f"[ERR] {r['file']} ->")
|
||||
@@ -113,10 +114,10 @@ def main():
|
||||
print(f"[OK] {r['file']}")
|
||||
|
||||
if any_errors:
|
||||
print('\nSome files have validation issues. Fix them or run with --dry-run to inspect.')
|
||||
print('\nCertains fichiers présentent des problèmes de validation. Corrigez-les ou lancez avec --dry-run pour inspecter.')
|
||||
sys.exit(2)
|
||||
else:
|
||||
print('\nAll files cleaned successfully.')
|
||||
print('\nTous les fichiers ont été nettoyés avec succès.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user