mirror of
https://github.com/Ladebeze66/llm_ticket3.git
synced 2025-12-13 12:36:50 +01:00
563 lines
25 KiB
Python
563 lines
25 KiB
Python
from ..base_agent import BaseAgent
|
|
import logging
|
|
import os
|
|
from typing import Dict, Any, List, Optional
|
|
from PIL import Image
|
|
from ..utils.pipeline_logger import sauvegarder_donnees
|
|
from utils.translate_utils import fr_to_en, en_to_fr
|
|
from datetime import datetime
|
|
import re
|
|
|
|
logger = logging.getLogger("AgentImageAnalyser")
|
|
|
|
class AgentImageAnalyser(BaseAgent):
|
|
"""
|
|
Agent for analyzing images and extracting relevant information.
|
|
Works in English and translates to French for compatibility.
|
|
"""
|
|
def __init__(self, llm, params: Optional[Dict[str, Any]] = None):
|
|
"""Initialise l'agent d'analyse d'images.
|
|
|
|
Args:
|
|
llm: Instance du modèle LLM à utiliser
|
|
params (Optional[Dict[str, Any]], optional): Paramètres de configuration. Defaults to None.
|
|
"""
|
|
super().__init__("AgentImageAnalyser", llm)
|
|
self.params = params or {
|
|
"temperature": 0.2,
|
|
"top_p": 0.8,
|
|
"max_tokens": 5000
|
|
}
|
|
|
|
self.instructions_analyse = (
|
|
"""
|
|
1. Objective Description
|
|
Describe precisely what the image shows:
|
|
- Software interface, menus, windows, tabs
|
|
- Error messages, system messages, code or script
|
|
- Software or module name/title if visible
|
|
- Clearly distinguish the complete name of tests/modules (for example, "Methylene blue test" instead of simply "blue test")
|
|
|
|
2. Key Technical Elements
|
|
Identify:
|
|
- Software versions or displayed modules
|
|
- Visible error codes
|
|
- Configurable parameters (text fields, sliders, dropdowns, checkboxes)
|
|
- Values displayed or pre-filled in fields
|
|
- Disabled, grayed out or hidden elements (often non-modifiable)
|
|
- Active/inactive buttons
|
|
- Reset or initialization buttons (often marked "RAZ" and not "PAZ")
|
|
- Specify if colored elements are part of the standard interface (e.g., always red button) or if they seem to be related to the problem
|
|
|
|
3. URLs and Links
|
|
- Identify and explicitly copy ALL URLs visible in the image
|
|
- Hyperlinks in blue or underlined text
|
|
- API endpoints, server addresses
|
|
- Format each URL on its own line for clarity: [URL] https://example.com
|
|
- For masked/shortened URLs, clearly indicate what text is displayed
|
|
|
|
4. Highlighted Elements
|
|
- Look for circled, framed, highlighted or arrowed areas
|
|
- These elements are often important for the client or support
|
|
- Explicitly mention their content and highlighting style
|
|
- Specifically check if error messages are visible at the bottom or top of the screen
|
|
|
|
5. Relationship with the Problem
|
|
- Establish the link between visible elements and the problem described in the ticket
|
|
- Indicate if components seem related to a misconfiguration or error
|
|
- Specify the complete name of the module/test concerned by the problem (for example "Methylene blue test (MB)" and not just "blue test")
|
|
- Identify if the user has access to the test screen but with errors, or if there is no access at all
|
|
|
|
6. Potential Answers
|
|
- Determine if the image provides elements of answer to a question asked in:
|
|
- The ticket title
|
|
- The problem description
|
|
- Try to extrapolate the precise technical context by observing the interface (e.g., the "blue test" mentioned by the client clearly corresponds to "methylene blue test (MB) - NF EN 933-9")
|
|
|
|
7. Link with the Discussion
|
|
- Check if the image echoes a step described in the discussion thread
|
|
- Note correspondences (e.g., same module, same error message as previously mentioned)
|
|
- Establish explicit connections between the vocabulary used by the client and what's visible in the interface
|
|
|
|
8. Broader Technical Context
|
|
- Identify the wider context of the application (laboratory, technical tests, standardized tests)
|
|
- Note any references to standards or norms (e.g., NF EN 933-9)
|
|
- Mention any visible codes or identifiers that might be useful (e.g., sample numbers)
|
|
|
|
Important Rules:
|
|
- Do NOT make ANY interpretation or diagnosis about possible causes
|
|
- Do NOT propose solutions or recommendations
|
|
- Remain strictly factual and objective, but make explicit links with terms used by the client
|
|
- Focus only on what is visible in the image
|
|
- Reproduce exact texts (e.g., error messages, parameter labels)
|
|
- Pay special attention to modifiable (interactive) and non-modifiable (grayed out) elements
|
|
- Systematically use the complete and precise name of modules and tests
|
|
- Verify correct reading of buttons and menus (beware of confusions like PAZ/RAZ)
|
|
- ALWAYS list URLs and links in a separate dedicated section
|
|
"""
|
|
)
|
|
|
|
self.system_prompt = (
|
|
"""
|
|
You are an expert in image analysis for BRG-Lab technical support for CBAO company.
|
|
Your mission is to analyze screenshots related to the support ticket context.
|
|
|
|
You must be extremely precise in your reading of interfaces and technical elements.
|
|
Clients often use abbreviated terms (like "blue test") while the interface shows the full term ("Methylene blue test"). You must make the connection between these terms.
|
|
|
|
Some elements in the interface may cause confusion:
|
|
- "RAZ" buttons (reset) are sometimes difficult to read
|
|
- Colored elements may be part of the standard interface (and not part of the problem)
|
|
- Error messages are often at the bottom of the screen and contain crucial information
|
|
- URLs and links must be explicitly captured and listed separately
|
|
|
|
Structure your image analysis factually:
|
|
{instructions}
|
|
|
|
Your analysis will be used as a factual element for a more complete technical report and to link the client's vocabulary with the actual technical elements.
|
|
|
|
IMPORTANT: All responses should be in English. Translation to French will be handled separately.
|
|
"""
|
|
).format(
|
|
instructions=self.instructions_analyse
|
|
)
|
|
|
|
# Collecteur de résultats pour traitement par lots (comme dans AgentImageSorter)
|
|
self.resultats = []
|
|
|
|
self._appliquer_config_locale()
|
|
logger.info("AgentImageAnalyser initialized")
|
|
|
|
def _appliquer_config_locale(self) -> None:
|
|
"""
|
|
Applies local configuration to the LLM model.
|
|
"""
|
|
if hasattr(self.llm, "prompt_system"):
|
|
self.llm.prompt_system = self.system_prompt
|
|
|
|
if hasattr(self.llm, "configurer"):
|
|
self.llm.configurer(**self.params)
|
|
|
|
def _verifier_image(self, image_path: str) -> bool:
|
|
"""
|
|
Checks if the image exists and is accessible
|
|
"""
|
|
try:
|
|
if not os.path.exists(image_path) or not os.access(image_path, os.R_OK):
|
|
return False
|
|
|
|
with Image.open(image_path) as img:
|
|
width, height = img.size
|
|
return width > 0 and height > 0
|
|
except Exception as e:
|
|
logger.error(f"Verification failed for {image_path}: {e}")
|
|
return False
|
|
|
|
def _extraire_urls(self, texte: str) -> List[str]:
|
|
"""
|
|
Extracts URLs from a text
|
|
|
|
Args:
|
|
texte: The text to analyze
|
|
|
|
Returns:
|
|
List of extracted URLs
|
|
"""
|
|
# Pattern to detect URLs (more complete than a simple http:// search)
|
|
url_pattern = r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+'
|
|
|
|
# Search in the text with a broader pattern to capture context
|
|
url_mentions = re.finditer(r'(?:URL|link|adresse|href|http)[^\n]*?(https?://[^\s\)\]\"\']+)', texte, re.IGNORECASE)
|
|
|
|
# List to store URLs with their context
|
|
urls = []
|
|
|
|
# Add URLs extracted with the generic pattern
|
|
for url in re.findall(url_pattern, texte):
|
|
if url not in urls:
|
|
urls.append(url)
|
|
|
|
# Add URLs extracted from the broader context
|
|
for match in url_mentions:
|
|
url = match.group(1)
|
|
if url not in urls:
|
|
urls.append(url)
|
|
|
|
return urls
|
|
|
|
def _construire_prompt(self, image_path: str, contexte: Dict[str, Any]) -> str:
|
|
"""
|
|
Construit le prompt pour l'analyse d'image avec contexte.
|
|
|
|
Args:
|
|
image_path: Chemin vers l'image à analyser
|
|
contexte: Contexte d'analyse du ticket et OCR
|
|
|
|
Returns:
|
|
Prompt formaté avec instructions et contexte
|
|
"""
|
|
image_name = os.path.basename(image_path)
|
|
|
|
# Extraire le contexte du ticket (résumé en anglais)
|
|
ticket_content_en = ""
|
|
if isinstance(contexte, dict):
|
|
# Vérifier les différentes structures possibles du contexte
|
|
if "response_en" in contexte:
|
|
ticket_content_en = contexte["response_en"]
|
|
elif "response" in contexte:
|
|
ticket_content_en = contexte["response"]
|
|
elif "analyse" in contexte:
|
|
# Structure de l'analyse de ticket
|
|
ticket_content_en = contexte["analyse"].get("en", "") or contexte["analyse"].get("analyse_en", "")
|
|
|
|
# Ajouter le texte OCR s'il est disponible
|
|
ocr_text = ""
|
|
if "ocr_text" in contexte:
|
|
ocr_text = contexte["ocr_text"]
|
|
elif "ocr_info" in contexte and isinstance(contexte["ocr_info"], dict):
|
|
ocr_text = contexte["ocr_info"].get("texte_en", "")
|
|
|
|
# Construire le prompt avec instructions précises
|
|
prompt = f"""[ENGLISH RESPONSE REQUESTED]
|
|
|
|
Analyze this image in the context of a technical support ticket.
|
|
|
|
IMAGE: {image_name}
|
|
|
|
"""
|
|
|
|
# Ajouter le texte OCR s'il est disponible
|
|
if ocr_text and len(ocr_text.strip()) > 10:
|
|
prompt += f"""OCR TEXT DETECTED IN IMAGE:
|
|
{ocr_text}
|
|
|
|
"""
|
|
|
|
# Ajouter le contexte du ticket s'il est disponible
|
|
if ticket_content_en:
|
|
prompt += f"""SUPPORT TICKET CONTEXT:
|
|
{ticket_content_en[:1500]}
|
|
|
|
"""
|
|
|
|
prompt += """INSTRUCTIONS:
|
|
1. Describe what is shown in this image in detail
|
|
2. Identify any error messages, technical information, or interface elements
|
|
3. Explain how this image relates to the support ticket context provided
|
|
4. Note any version numbers, status indicators, or dates visible
|
|
5. Extract specific technical details that might help diagnose the issue
|
|
|
|
If the image contains text, code, or error messages, transcribe all important parts.
|
|
Structure your analysis clearly with headers and bullet points.
|
|
"""
|
|
|
|
logger.debug(f"Prompt construit pour {image_name} avec OCR: {bool(ocr_text)} et contexte ticket: {bool(ticket_content_en)}")
|
|
return prompt
|
|
|
|
def _extraire_ticket_id_depuis_path(self, path: str) -> str:
|
|
"""Extrait l'ID du ticket depuis le chemin de l'image.
|
|
|
|
Args:
|
|
path (str): Chemin de l'image
|
|
|
|
Returns:
|
|
str: ID du ticket ou 'unknown' si non trouvé
|
|
"""
|
|
try:
|
|
# Recherche un pattern comme T12345 dans le chemin
|
|
match = re.search(r'T\d+', path)
|
|
if match:
|
|
return match.group(0)
|
|
except Exception as e:
|
|
logger.error(f"Erreur lors de l'extraction de l'ID du ticket: {e}")
|
|
return "unknown"
|
|
|
|
def executer(self, image_path: str, contexte: Optional[dict] = None) -> dict:
|
|
"""
|
|
Analyse une image et extrait les informations pertinentes.
|
|
|
|
Args:
|
|
image_path: Chemin vers l'image à analyser
|
|
contexte: Contexte optionnel (texte OCR, analyse ticket, etc)
|
|
|
|
Returns:
|
|
Dictionnaire contenant les résultats d'analyse
|
|
"""
|
|
logger.info(f"Analyzing image: {image_path}")
|
|
|
|
try:
|
|
if not self._verifier_image(image_path):
|
|
return self._erreur("Image inaccessible ou invalide", image_path)
|
|
|
|
# Construire le prompt avec le contexte
|
|
prompt = self._construire_prompt(image_path, contexte or {})
|
|
|
|
# Analyser l'image avec le LLM
|
|
if not hasattr(self.llm, "interroger_avec_image"):
|
|
return self._erreur("Le modèle ne supporte pas l'analyse d'images", image_path)
|
|
|
|
logger.info(f"[LANGUE] Envoi d'une requête en anglais au modèle avec une image: {os.path.basename(image_path)}")
|
|
logger.info(f"[LANGUE] Taille du prompt en anglais: {len(prompt)} caractères")
|
|
|
|
response = self.llm.interroger_avec_image(image_path, prompt)
|
|
|
|
logger.info(f"[LANGUE] Réponse reçue du modèle en anglais: {len(response)} caractères")
|
|
|
|
if self._verifier_reponse_invalide(response):
|
|
return self._erreur("Réponse du modèle invalide", image_path, response)
|
|
|
|
# Extraire le ticket_id
|
|
ticket_id = self._extraire_ticket_id(image_path, contexte or {})
|
|
|
|
# Nettoyer le nom du modèle pour éviter les doublons
|
|
model_name = getattr(self.llm, "pipeline_normalized_name", None)
|
|
if not model_name:
|
|
# Si pipeline_normalized_name n'est pas disponible, utiliser le nom du modèle
|
|
model_name = getattr(self.llm, "modele", "llama3-2-vision-90b-instruct")
|
|
# Normaliser manuellement
|
|
model_name = model_name.replace(".", "-").replace(":", "-").replace("_", "-")
|
|
|
|
logger.info(f"Model name used for logging: {model_name}")
|
|
logger.debug(f"Nom du modèle avant normalisation: {getattr(self.llm, 'modele', 'inconnu')}")
|
|
logger.debug(f"Nom du modèle normalisé: {model_name}")
|
|
|
|
|
|
# Traduire la réponse en français
|
|
logger.info(f"[TRADUCTION] Traduction de la réponse d'analyse d'image de EN vers FR")
|
|
logger.info(f"[TRADUCTION] Taille de la réponse en anglais: {len(response)} caractères")
|
|
response_fr = en_to_fr(response)
|
|
logger.info(f"[TRADUCTION] Taille de la réponse traduite en français: {len(response_fr)} caractères")
|
|
|
|
# Construire le résultat
|
|
result = {
|
|
"timestamp": datetime.now().isoformat(),
|
|
"image": os.path.basename(image_path),
|
|
"ticket_id": ticket_id,
|
|
"analyse": {
|
|
"en": response,
|
|
"fr": response_fr
|
|
},
|
|
"model_info": {
|
|
"model": model_name,
|
|
**self.params
|
|
}
|
|
}
|
|
|
|
# Extraire les URLs trouvées dans la réponse
|
|
urls = self._extraire_urls(response)
|
|
if urls:
|
|
logger.info(f"[ANALYSE] {len(urls)} URLs extraites de l'analyse: {urls}")
|
|
result["urls"] = urls
|
|
|
|
# Ajouter au collecteur de résultats
|
|
self.resultats.append(result)
|
|
|
|
logger.debug(f"Résultat de l'analyse pour l'image {image_path}: {result}")
|
|
logger.info(f"[LANGUES] Résultat d'analyse disponible en deux langues: EN et FR")
|
|
|
|
return result
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error analyzing image {image_path}: {e}")
|
|
return self._erreur(f"Erreur inattendue: {str(e)}", image_path)
|
|
|
|
def _corriger_termes_courants(self, texte: str) -> str:
|
|
"""
|
|
Corrects commonly misinterpreted terms by the model.
|
|
"""
|
|
corrections = {
|
|
"PAZ": "RAZ",
|
|
"Essai bleu": "Essai au bleu de méthylène",
|
|
"essai bleu": "essai au bleu de méthylène",
|
|
"Essai au bleu": "Essai au bleu de méthylène",
|
|
"Methylene blue test": "Essai au bleu de méthylène",
|
|
"Blue test": "Essai au bleu de méthylène"
|
|
}
|
|
|
|
for terme_incorrect, terme_correct in corrections.items():
|
|
texte = texte.replace(terme_incorrect, terme_correct)
|
|
|
|
return texte
|
|
|
|
def _erreur(self, message: str, path: str, details: Any = None) -> Dict[str, Any]:
|
|
"""
|
|
Crée un dictionnaire d'erreur standardisé.
|
|
|
|
Args:
|
|
message: Message d'erreur
|
|
path: Chemin du fichier concerné
|
|
details: Détails supplémentaires de l'erreur (optionnel)
|
|
|
|
Returns:
|
|
Dictionnaire contenant les informations d'erreur
|
|
"""
|
|
error_dict = {
|
|
"timestamp": self._get_timestamp(),
|
|
"success": False,
|
|
"error": message,
|
|
"image": os.path.basename(path),
|
|
"metadata": {
|
|
"error_details": details if details else {},
|
|
"source_agent": self.nom
|
|
}
|
|
}
|
|
logger.error(f"Erreur: {message} pour {path}")
|
|
return error_dict
|
|
|
|
def _get_timestamp(self) -> str:
|
|
"""Returns a timestamp in YYYYMMDD_HHMMSS format"""
|
|
return datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
|
|
def _extraire_ticket_id(self, image_path: str, contexte: Dict[str, Any]) -> str:
|
|
"""
|
|
Extrait l'ID du ticket à partir du chemin de l'image ou du contexte.
|
|
|
|
Args:
|
|
image_path: Chemin vers l'image
|
|
contexte: Contexte d'analyse du ticket
|
|
|
|
Returns:
|
|
ID du ticket ou "UNKNOWN" si non trouvé
|
|
"""
|
|
# D'abord, chercher dans le contexte
|
|
if isinstance(contexte, dict):
|
|
if "metadata" in contexte and "ticket_id" in contexte["metadata"]:
|
|
return contexte["metadata"]["ticket_id"]
|
|
if "ticket_id" in contexte:
|
|
return contexte["ticket_id"]
|
|
|
|
# Ensuite, chercher dans le chemin de l'image
|
|
parts = image_path.split(os.path.sep)
|
|
for part in parts:
|
|
# Format T12345
|
|
if part.startswith("T") and part[1:].isdigit():
|
|
return part
|
|
# Format ticket_T12345
|
|
if part.startswith("ticket_T"):
|
|
return part.replace("ticket_", "")
|
|
|
|
return "UNKNOWN"
|
|
|
|
def _error_response(self, message: str, ticket_id: str = "UNKNOWN") -> Dict[str, Any]:
|
|
"""
|
|
Crée une réponse d'erreur standardisée.
|
|
|
|
Args:
|
|
message: Message d'erreur
|
|
ticket_id: ID du ticket
|
|
|
|
Returns:
|
|
Dictionnaire avec la réponse d'erreur formatée
|
|
"""
|
|
return {
|
|
"analyse": f"ERREUR: {message}",
|
|
"analyse_en": f"ERROR: {message}",
|
|
"error": True,
|
|
"metadata": {
|
|
"timestamp": self._get_timestamp(),
|
|
"error": True,
|
|
"ticket_id": ticket_id,
|
|
"source_agent": self.nom
|
|
}
|
|
}
|
|
|
|
def _verifier_reponse_invalide(self, response: str) -> bool:
|
|
"""
|
|
Vérifie si la réponse du modèle est invalide ou inappropriée
|
|
|
|
Args:
|
|
response: Réponse du modèle à analyser
|
|
|
|
Returns:
|
|
True si la réponse est invalide, False sinon
|
|
"""
|
|
response_lower = response.lower()
|
|
|
|
# Vérifier les marqueurs d'échec courants
|
|
invalid_markers = [
|
|
"i cannot", "unable to", "i'm unable", "i am unable",
|
|
"i don't see", "i do not see", "i can't see", "cannot see",
|
|
"sorry, i cannot", "i apologize", "not able to"
|
|
]
|
|
|
|
# Si la réponse est vide ou trop courte
|
|
if not response or len(response.strip()) < 20:
|
|
return True
|
|
|
|
# Si la réponse contient des marqueurs d'échec
|
|
for marker in invalid_markers:
|
|
if marker in response_lower:
|
|
# Vérifier qu'il s'agit bien d'un échec global et non d'une réponse légitime
|
|
# qui inclut ces termes dans un contexte différent
|
|
context_words = ["but i can", "however", "nevertheless", "although", "can describe"]
|
|
has_context = any(context in response_lower for context in context_words)
|
|
|
|
if not has_context and marker in response_lower[:100]:
|
|
return True
|
|
|
|
return False
|
|
|
|
def sauvegarder_resultats(self) -> None:
|
|
"""
|
|
Sauvegarde tous les résultats collectés en garantissant leur accumulation.
|
|
Utilise un format de liste pour maintenir les multiples résultats.
|
|
"""
|
|
logger.info(f"Sauvegarde de {len(self.resultats)} résultats d'analyse d'images")
|
|
|
|
if not self.resultats:
|
|
return
|
|
|
|
# Récupérer le ticket_id du premier résultat
|
|
ticket_id = self.resultats[0].get("ticket_id", self.resultats[0].get("metadata", {}).get("ticket_id", "UNKNOWN"))
|
|
|
|
try:
|
|
# Obtenir directement le nom normalisé du modèle depuis l'instance LLM
|
|
normalized_model_name = getattr(self.llm, "pipeline_normalized_name", None)
|
|
if normalized_model_name:
|
|
logger.info(f"Utilisation du nom de modèle normalisé depuis LLM: {normalized_model_name}")
|
|
else:
|
|
# Fallback : utiliser le nom du modèle de l'instance LLM
|
|
normalized_model_name = getattr(self.llm, "modele", "llama3-vision-90b-instruct")
|
|
# Normaliser manuellement
|
|
normalized_model_name = normalized_model_name.replace(".", "-").replace(":", "-").replace("_", "-")
|
|
logger.info(f"Fallback : utilisation du nom de modèle normalisé manuellement: {normalized_model_name}")
|
|
|
|
# Normaliser les noms de modèles dans tous les résultats
|
|
for result in self.resultats:
|
|
if "model_info" in result and "model" in result["model_info"]:
|
|
# Utiliser le nom de modèle normalisé pour tous les résultats
|
|
result["model_info"]["model"] = normalized_model_name
|
|
logger.debug(f"Nom de modèle défini pour un résultat: {normalized_model_name}")
|
|
|
|
# Ajouter un log pour voir le premier résultat avec le modèle normalisé
|
|
if self.resultats and "model_info" in self.resultats[0]:
|
|
logger.info(f"Modèle utilisé pour sauvegarder les résultats: {self.resultats[0]['model_info'].get('model', 'non défini')}")
|
|
|
|
# Sauvegarder en mode liste pour accumuler les résultats
|
|
sauvegarder_donnees(
|
|
ticket_id=ticket_id,
|
|
step_name="analyse_image",
|
|
data=self.resultats,
|
|
base_dir=None,
|
|
is_resultat=True
|
|
)
|
|
print(f"Sauvegarde groupée de {len(self.resultats)} résultats d'analyse d'images")
|
|
|
|
# Vérifier si les fichiers ont été créés avec le bon nom
|
|
from os import path, listdir
|
|
rapport_dir = path.join("output", f"ticket_{ticket_id}")
|
|
if path.exists(rapport_dir):
|
|
extractions = [d for d in listdir(rapport_dir) if path.isdir(path.join(rapport_dir, d)) and d.startswith(ticket_id)]
|
|
if extractions:
|
|
extraction_path = path.join(rapport_dir, sorted(extractions, reverse=True)[0])
|
|
pipeline_dir = path.join(extraction_path, f"{ticket_id}_rapports", "pipeline")
|
|
if path.exists(pipeline_dir):
|
|
files = [f for f in listdir(pipeline_dir) if f.startswith("analyse_image_") and f.endswith("_results.json")]
|
|
logger.info(f"Fichiers d'analyse d'images trouvés après sauvegarde: {files}")
|
|
|
|
# Réinitialiser la liste après la sauvegarde
|
|
self.resultats = []
|
|
except Exception as e:
|
|
logger.error(f"Erreur lors de la sauvegarde des résultats d'analyse d'images : {e}")
|
|
logger.exception("Détails de l'erreur:")
|
|
print(f"Erreur lors de la sauvegarde des résultats : {e}") |