La Vectorisation de Données : De la Théorie à la Pratique

from sentence_transformers import SentenceTransformer
import numpy as np
from qdrant_client import QdrantClient

# 1. Initialisation du modèle
model = SentenceTransformer('all-MiniLM-L6-v2')  # 384 dimensions

# 2. Préprocessing
texts = ["Paris est la capitale de France", "Berlin est en Allemagne"]
cleaned_texts = [t.strip().lower() for t in texts]

# 3-4. Vectorisation
vectors = model.encode(cleaned_texts, normalize_embeddings=True)
print(vectors.shape)  # (2, 384)

# 5. Post-processing (normalisation L2 déjà faite)
assert np.allclose(np.linalg.norm(vectors[0]), 1.0)

# 6. Stockage dans Qdrant
client = QdrantClient(":memory:")
client.create_collection(
    collection_name="documents",
    vectors_config={"size": 384, "distance": "Cosine"}
)

import pandas as pd
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer

# Dataset exemple
df = pd.DataFrame({
    'age': [25, 45, 35],
    'salary': [30000, 80000, 55000],
    'country': ['France', 'USA', 'France'],
    'category': ['A', 'B', 'A']
})

# Définition du preprocessor
preprocessor = ColumnTransformer(
    transformers=[
        ('num', StandardScaler(), ['age', 'salary']),
        ('cat', OneHotEncoder(drop='first'), ['country', 'category'])
    ])

# Transformation
vectors = preprocessor.fit_transform(df)
print(vectors.shape)  # (3, 5) -> 2 num + 3 cat (one-hot)
print(vectors)
# [[-1.22  -1.22   0.     0.   ]
#  [ 1.22   1.22   1.     1.   ]
#  [ 0.     0.     0.     0.   ]]

import pandas as pd
from pandas import json_normalize

json_data = {
    "user": {"id": 123, "name": "Alice"},
    "purchase": {"amount": 99.99, "items": ["book", "pen"]}
}

# Flatten
df = json_normalize(json_data)
print(df.columns)
# ['user.id', 'user.name', 'purchase.amount', 'purchase.items']

# Puis vectorisation classique (One-Hot, StandardScaler, etc.)

from sklearn.feature_extraction.text import CountVectorizer

corpus = [
    "Paris est la capitale de France",
    "Berlin est la capitale de Allemagne",
    "Madrid est en Espagne"
]

vectorizer = CountVectorizer()
X = vectorizer.fit_transform(corpus)

print(vectorizer.get_feature_names_out())
# ['allemagne' 'berlin' 'capitale' 'de' 'en' 'espagne' 'est' 'france' 'la' 'madrid' 'paris']

print(X.toarray())
# [[0 0 1 1 0 0 1 1 1 0 1]  # Doc 1
#  [1 1 1 1 0 0 1 0 1 0 0]  # Doc 2
#  [0 0 0 0 1 1 1 0 0 1 0]] # Doc 3

from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer(max_features=1000, ngram_range=(1,2))
X = vectorizer.fit_transform(corpus)

print(X.shape)  # (3, 16) - 16 features uniques (unigrams + bigrams)
print(X[0])  # Sparse vector avec scores TF-IDF
# Mots fréquents partout ("est", "la") ont scores faibles
# Mots rares ("Paris", "France") ont scores élevés

from gensim.models import Word2Vec
import numpy as np

# Corpus tokenizé
sentences = [
    ["paris", "capitale", "france"],
    ["berlin", "capitale", "allemagne"],
    ["madrid", "capitale", "espagne"],
    ["paris", "tour", "eiffel"],
    ["berlin", "porte", "brandebourg"]
]

# Entraînement
model = Word2Vec(sentences, vector_size=100, window=3, min_count=1, epochs=100)

# Vecteur d'un mot
vec_paris = model.wv['paris']
print(vec_paris.shape)  # (100,)

# Similarités
print(model.wv.most_similar('paris', topn=3))
# [('berlin', 0.89), ('madrid', 0.87), ('capitale', 0.76)]

# Algèbre vectorielle célèbre
result = model.wv.most_similar(positive=['paris', 'allemagne'], negative=['france'])
print(result[0])  # ('berlin', 0.92) - analogie "roi-homme+femme=reine"

import numpy as np
from gensim.models.fasttext import FastText

# Modèle FastText
model = FastText(sentences, vector_size=100, window=3, min_count=1)

# Vectoriser un document = moyenne des vecteurs de mots
def document_vector(doc, model):
    vectors = [model.wv[word] for word in doc if word in model.wv]
    if len(vectors) == 0:
        return np.zeros(model.vector_size)
    return np.mean(vectors, axis=0)

doc = ["paris", "est", "belle"]
vec = document_vector(doc, model)
print(vec.shape)  # (100,)

from transformers import AutoTokenizer, AutoModel
import torch

# Chargement modèle BERT
model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# Texte à vectoriser
text = "Paris est la capitale de France"

# Tokenization
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)

# Inférence
with torch.no_grad():
    outputs = model(**inputs)

# Extraction embedding [CLS] (représentation de la phrase entière)
sentence_embedding = outputs.last_hidden_state[:, 0, :]
print(sentence_embedding.shape)  # (1, 768)

# Alternative : moyenne de tous les tokens
mean_embedding = outputs.last_hidden_state.mean(dim=1)
print(mean_embedding.shape)  # (1, 768)

from sentence_transformers import SentenceTransformer
import numpy as np

# Modèles populaires :
# - all-MiniLM-L6-v2 : 384D, 22M params, rapide (5ms/doc CPU)
# - all-mpnet-base-v2 : 768D, 110M params, meilleure qualité (15ms/doc CPU)
# - paraphrase-multilingual : Support 50+ langues

model = SentenceTransformer('all-MiniLM-L6-v2')

# Vectorisation (batch processing automatique)
documents = [
    "Paris est la capitale de France",
    "Berlin est la capitale de l'Allemagne",
    "J'aime le chocolat"
]

embeddings = model.encode(documents, normalize_embeddings=True)
print(embeddings.shape)  # (3, 384)

# Similarité cosinus
from sklearn.metrics.pairwise import cosine_similarity
sim_matrix = cosine_similarity(embeddings)
print(sim_matrix)
# [[1.   0.87 0.12]  # Paris-Berlin très similaires
#  [0.87 1.   0.09]  # Paris-Chocolat peu similaires
#  [0.12 0.09 1.  ]]

from openai import OpenAI
import os

client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

def get_embedding(text, model="text-embedding-3-small"):
    text = text.replace("\n", " ")
    response = client.embeddings.create(input=[text], model=model)
    return response.data[0].embedding

embedding = get_embedding("Paris est la capitale de France")
print(len(embedding))  # 1536
print(embedding[:5])  # [0.023, -0.018, 0.045, -0.012, 0.067]

from skimage.feature import hog
from skimage import io
import numpy as np

# Charger image
image = io.imread('cat.jpg', as_gray=True)

# Extraction HOG
features, hog_image = hog(
    image,
    orientations=9,
    pixels_per_cell=(8, 8),
    cells_per_block=(2, 2),
    visualize=True
)

print(features.shape)  # (7524,) - vecteur sparse de features

import torch
import torchvision.models as models
import torchvision.transforms as transforms
from PIL import Image

# Charger ResNet50 pré-entraîné
model = models.resnet50(pretrained=True)
model.eval()

# Retirer la dernière couche (classifier) pour garder features
feature_extractor = torch.nn.Sequential(*list(model.children())[:-1])

# Preprocessing (requis pour ImageNet)
preprocess = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])

# Charger et préprocesser image
img = Image.open('cat.jpg')
input_tensor = preprocess(img).unsqueeze(0)  # (1, 3, 224, 224)

# Extraction features
with torch.no_grad():
    features = feature_extractor(input_tensor)

features = features.squeeze()  # (2048,)
print(features.shape)  # torch.Size([2048])

from transformers import ViTImageProcessor, ViTModel
from PIL import Image
import torch

# Charger modèle ViT pré-entraîné
processor = ViTImageProcessor.from_pretrained('google/vit-base-patch16-224')
model = ViTModel.from_pretrained('google/vit-base-patch16-224')

# Charger image
image = Image.open('cat.jpg')

# Preprocessing
inputs = processor(images=image, return_tensors="pt")

# Extraction features
with torch.no_grad():
    outputs = model(**inputs)

# [CLS] token embedding (représentation image entière)
image_embedding = outputs.last_hidden_state[:, 0, :]
print(image_embedding.shape)  # (1, 768)

# Alternative : pooler output
pooled = outputs.pooler_output
print(pooled.shape)  # (1, 768)

import torch
import clip
from PIL import Image

# Charger modèle CLIP
device = "cuda" if torch.cuda.is_available() else "cpu"
model, preprocess = clip.load("ViT-B/32", device=device)

# Images candidates
images = [
    preprocess(Image.open("cat.jpg")).unsqueeze(0).to(device),
    preprocess(Image.open("dog.jpg")).unsqueeze(0).to(device),
    preprocess(Image.open("car.jpg")).unsqueeze(0).to(device)
]
images = torch.cat(images)

# Queries texte
texts = clip.tokenize(["a photo of a cat", "a photo of a dog"]).to(device)

# Encodage
with torch.no_grad():
    image_features = model.encode_image(images)  # (3, 512)
    text_features = model.encode_text(texts)      # (2, 512)

    # Normalisation L2
    image_features /= image_features.norm(dim=-1, keepdim=True)
    text_features /= text_features.norm(dim=-1, keepdim=True)

    # Similarité cosinus
    similarity = (100.0 * text_features @ image_features.T).softmax(dim=-1)

print(similarity)
# tensor([[0.95, 0.03, 0.02],  # "a photo of a cat" -> cat.jpg (95%)
#         [0.05, 0.92, 0.03]]) # "a photo of a dog" -> dog.jpg (92%)

import librosa
import numpy as np

# Charger audio
audio_path = 'speech.wav'
y, sr = librosa.load(audio_path, sr=22050)  # y=waveform, sr=sample rate

# Extraction MFCC
mfccs = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=40)
print(mfccs.shape)  # (40, T) - 40 coefficients, T frames temporels

# Statistiques pour vectorisation fixe
mfcc_mean = np.mean(mfccs, axis=1)  # (40,)
mfcc_std = np.std(mfccs, axis=1)    # (40,)
vector = np.concatenate([mfcc_mean, mfcc_std])  # (80,)
print(vector.shape)  # (80,) - vecteur final fixe

import librosa
import librosa.display
import matplotlib.pyplot as plt

# Extraction Mel-Spectrogram
mel_spec = librosa.feature.melspectrogram(y=y, sr=sr, n_mels=128, fmax=8000)
mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max)

print(mel_spec_db.shape)  # (128, T) - 128 bandes mel, T frames

# Utiliser comme "image" pour CNN
# Shape: (128, T) → (1, 128, T) pour PyTorch (channels, height, width)
import torch
mel_tensor = torch.from_numpy(mel_spec_db).unsqueeze(0)
print(mel_tensor.shape)  # (1, 128, T)

from transformers import Wav2Vec2Processor, Wav2Vec2Model
import torch
import librosa

# Charger modèle
processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base")
model = Wav2Vec2Model.from_pretrained("facebook/wav2vec2-base")

# Charger audio
audio, sr = librosa.load('speech.wav', sr=16000)  # Wav2Vec2 requiert 16kHz

# Preprocessing
inputs = processor(audio, sampling_rate=sr, return_tensors="pt", padding=True)

# Extraction features
with torch.no_grad():
    outputs = model(**inputs)

# Embeddings (moyenne sur temps)
embeddings = outputs.last_hidden_state.mean(dim=1)  # (1, 768)
print(embeddings.shape)  # (1, 768)

import whisper
import torch
import librosa

# Charger modèle Whisper
model = whisper.load_model("base")  # tiny/base/small/medium/large

# Charger audio
audio, sr = librosa.load('speech.wav', sr=16000)

# Padding/Truncation à 30s (Whisper limite)
audio = whisper.pad_or_trim(audio)

# Mel-spectrogram (fait par Whisper)
mel = whisper.log_mel_spectrogram(audio).to(model.device)

# Extraction embeddings depuis encoder
with torch.no_grad():
    embeddings = model.embed_audio(mel.unsqueeze(0))

print(embeddings.shape)  # (1, 1500, 512) - séquence de frames

# Pooling pour vecteur fixe
audio_vector = embeddings.mean(dim=1)  # (1, 512)
print(audio_vector.shape)

from transformers import pipeline

# Pipeline pré-entraîné pour classification audio
classifier = pipeline(
    "audio-classification",
    model="superb/wav2vec2-base-superb-er"  # Emotion Recognition
)

# Classification
result = classifier('speech.wav')
print(result)
# [{'label': 'happy', 'score': 0.87},
#  {'label': 'neutral', 'score': 0.09}, ...]

# Créer environnement virtuel
python -m venv venv
source venv/bin/activate  # Linux/Mac
# ou : venv\Scripts\activate  # Windows

# Packages essentiels
pip install numpy pandas scikit-learn

# NLP / Texte
pip install sentence-transformers transformers torch
pip install openai  # Pour OpenAI embeddings API

# Vision / Images
pip install torchvision pillow opencv-python
pip install timm  # PyTorch Image Models (EfficientNet, etc.)

# Audio
pip install librosa soundfile
pip install git+https://github.com/openai/whisper.git

# Bases vectorielles
pip install qdrant-client chromadb pinecone-client

# Optimisation et monitoring
pip install onnx onnxruntime tqdm

import torch

print(f"CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
    print(f"GPU: {torch.cuda.get_device_name(0)}")
    print(f"Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")

# Pour utiliser GPU avec Sentence Transformers
from sentence_transformers import SentenceTransformer
model = SentenceTransformer('all-MiniLM-L6-v2', device='cuda')  # ou 'cpu'

# config.yaml
text:
  model_name: "all-MiniLM-L6-v2"
  batch_size: 32
  max_length: 512
  normalize: true

image:
  model_name: "resnet50"
  image_size: 224
  batch_size: 16

audio:
  model_name: "whisper-base"
  sample_rate: 16000

vector_db:
  provider: "qdrant"  # ou "pinecone", "chromadb"
  host: "localhost"
  port: 6333
  collection_name: "documents"

from sentence_transformers import SentenceTransformer
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct
import uuid
from typing import List, Dict
from tqdm import tqdm

class TextVectorizer:
    def __init__(self, model_name='all-MiniLM-L6-v2', batch_size=32):
        self.model = SentenceTransformer(model_name)
        self.batch_size = batch_size
        self.vector_size = self.model.get_sentence_embedding_dimension()

    def preprocess(self, text: str) -> str:
        """Nettoyage texte basique"""
        text = text.strip()
        text = ' '.join(text.split())  # Normaliser espaces
        return text

    def vectorize(self, texts: List[str]) -> List[List[float]]:
        """Vectorisation batch avec barre de progression"""
        cleaned_texts = [self.preprocess(t) for t in texts]
        vectors = self.model.encode(
            cleaned_texts,
            batch_size=self.batch_size,
            normalize_embeddings=True,
            show_progress_bar=True
        )
        return vectors.tolist()

    def vectorize_and_store(self, documents: List[Dict], collection_name: str):
        """
        Vectorise documents et stocke dans Qdrant

        Args:
            documents: Liste de dicts avec keys 'text' et 'metadata'
            collection_name: Nom collection Qdrant
        """
        # Connexion Qdrant
        client = QdrantClient(host="localhost", port=6333)

        # Création collection
        client.recreate_collection(
            collection_name=collection_name,
            vectors_config=VectorParams(
                size=self.vector_size,
                distance=Distance.COSINE
            )
        )

        # Vectorisation
        texts = [doc['text'] for doc in documents]
        vectors = self.vectorize(texts)

        # Upload vers Qdrant
        points = [
            PointStruct(
                id=str(uuid.uuid4()),
                vector=vector,
                payload={
                    'text': doc['text'],
                    **doc.get('metadata', {})
                }
            )
            for doc, vector in zip(documents, vectors)
        ]

        # Upload par batches
        for i in tqdm(range(0, len(points), 100), desc="Uploading to Qdrant"):
            batch = points[i:i+100]
            client.upsert(
                collection_name=collection_name,
                points=batch
            )

        print(f"Vectorized and stored {len(documents)} documents")
        return client

# Utilisation
if __name__ == "__main__":
    # Documents exemple
    documents = [
        {
            "text": "Paris est la capitale de France",
            "metadata": {"category": "geography", "lang": "fr"}
        },
        {
            "text": "La Tour Eiffel mesure 330 mètres",
            "metadata": {"category": "monument", "lang": "fr"}
        },
        {
            "text": "Python est un langage de programmation",
            "metadata": {"category": "tech", "lang": "fr"}
        }
    ]

    # Vectorisation
    vectorizer = TextVectorizer()
    client = vectorizer.vectorize_and_store(documents, "my_docs")

    # Recherche sémantique
    query = "Quelle est la hauteur de la tour Eiffel ?"
    query_vector = vectorizer.vectorize([query])[0]

    results = client.search(
        collection_name="my_docs",
        query_vector=query_vector,
        limit=3
    )

    print("\nRésultats recherche:")
    for result in results:
        print(f"Score: {result.score:.3f} - {result.payload['text']}")

import torch
import torchvision.models as models
import torchvision.transforms as transforms
from PIL import Image
import os
from typing import List
import numpy as np
from tqdm import tqdm

class ImageVectorizer:
    def __init__(self, model_name='resnet50', device='cuda'):
        self.device = device if torch.cuda.is_available() else 'cpu'

        # Charger modèle
        if model_name == 'resnet50':
            model = models.resnet50(pretrained=True)
            self.feature_extractor = torch.nn.Sequential(
                *list(model.children())[:-1]  # Retirer FC layer
            )
            self.vector_size = 2048
        elif model_name == 'efficientnet_b0':
            model = models.efficientnet_b0(pretrained=True)
            self.feature_extractor = model
            self.vector_size = 1280

        self.feature_extractor.to(self.device)
        self.feature_extractor.eval()

        # Preprocessing ImageNet standard
        self.transform = transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize(
                mean=[0.485, 0.456, 0.406],
                std=[0.229, 0.224, 0.225]
            )
        ])

    def load_image(self, image_path: str) -> torch.Tensor:
        """Charge et préprocess une image"""
        img = Image.open(image_path).convert('RGB')
        return self.transform(img)

    def vectorize_single(self, image_path: str) -> np.ndarray:
        """Vectorise une seule image"""
        img_tensor = self.load_image(image_path).unsqueeze(0)
        img_tensor = img_tensor.to(self.device)

        with torch.no_grad():
            features = self.feature_extractor(img_tensor)

        vector = features.squeeze().cpu().numpy()
        # Normalisation L2
        vector = vector / np.linalg.norm(vector)
        return vector

    def vectorize_batch(self, image_paths: List[str], batch_size=16) -> np.ndarray:
        """Vectorisation batch efficace"""
        all_vectors = []

        for i in tqdm(range(0, len(image_paths), batch_size), desc="Vectorizing images"):
            batch_paths = image_paths[i:i+batch_size]

            # Charger batch
            batch_tensors = []
            for path in batch_paths:
                try:
                    tensor = self.load_image(path)
                    batch_tensors.append(tensor)
                except Exception as e:
                    print(f"Error loading {path}: {e}")
                    continue

            if len(batch_tensors) == 0:
                continue

            # Stack et vers GPU
            batch = torch.stack(batch_tensors).to(self.device)

            # Extraction features
            with torch.no_grad():
                features = self.feature_extractor(batch)

            # Normalisation
            vectors = features.squeeze().cpu().numpy()
            if len(vectors.shape) == 1:  # Single image
                vectors = vectors.reshape(1, -1)

            vectors = vectors / np.linalg.norm(vectors, axis=1, keepdims=True)
            all_vectors.append(vectors)

        return np.vstack(all_vectors)

    def find_similar_images(self, query_path: str, image_paths: List[str], top_k=5):
        """Trouve images les plus similaires"""
        # Vectoriser query
        query_vector = self.vectorize_single(query_path)

        # Vectoriser corpus
        corpus_vectors = self.vectorize_batch(image_paths)

        # Similarité cosinus
        similarities = np.dot(corpus_vectors, query_vector)

        # Top-K
        top_indices = np.argsort(similarities)[::-1][:top_k]

        results = [
            (image_paths[idx], similarities[idx])
            for idx in top_indices
        ]

        return results

# Utilisation
if __name__ == "__main__":
    vectorizer = ImageVectorizer(model_name='resnet50')

    # Vectoriser dossier d'images
    image_folder = "./images/"
    image_paths = [os.path.join(image_folder, f) for f in os.listdir(image_folder) if f.endswith(('.jpg', '.png'))]

    # Recherche images similaires
    query_image = "./query/cat.jpg"
    similar_images = vectorizer.find_similar_images(query_image, image_paths, top_k=5)

    print("\nImages similaires:")
    for img_path, score in similar_images:
        print(f"Score: {score:.3f} - {img_path}")

from dataclasses import dataclass
from typing import List, Dict, Optional, Union
import logging
from pathlib import Path
import time
import numpy as np
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, VectorParams, PointStruct
import uuid

# Import vectorizers
from text_vectorizer import TextVectorizer
from image_vectorizer import ImageVectorizer

# Configuration logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

@dataclass
class Document:
    """Représentation unifiée d'un document"""
    id: str
    text: Optional[str] = None
    image_path: Optional[str] = None
    metadata: Dict = None

class ProductionPipeline:
    def __init__(self, config: Dict):
        self.config = config

        # Initialisation vectorizers
        self.text_vectorizer = TextVectorizer(
            model_name=config['text']['model_name'],
            batch_size=config['text']['batch_size']
        )

        self.image_vectorizer = ImageVectorizer(
            model_name=config['image']['model_name']
        )

        # Qdrant client
        self.db_client = QdrantClient(
            host=config['vector_db']['host'],
            port=config['vector_db']['port']
        )

        # Stats
        self.stats = {
            'processed': 0,
            'errors': 0,
            'total_time': 0
        }

    def setup_collections(self):
        """Crée collections Qdrant"""
        # Collection texte
        self.db_client.recreate_collection(
            collection_name="text_vectors",
            vectors_config=VectorParams(
                size=self.text_vectorizer.vector_size,
                distance=Distance.COSINE
            )
        )

        # Collection images
        self.db_client.recreate_collection(
            collection_name="image_vectors",
            vectors_config=VectorParams(
                size=self.image_vectorizer.vector_size,
                distance=Distance.COSINE
            )
        )

        logger.info("Collections created successfully")

    def process_document(self, doc: Document, retry=3) -> bool:
        """Traite un document avec retry logic"""
        for attempt in range(retry):
            try:
                start_time = time.time()

                # Vectorisation texte
                if doc.text:
                    text_vector = self.text_vectorizer.vectorize([doc.text])[0]
                    self.db_client.upsert(
                        collection_name="text_vectors",
                        points=[PointStruct(
                            id=doc.id,
                            vector=text_vector,
                            payload={'text': doc.text, **(doc.metadata or {})}
                        )]
                    )

                # Vectorisation image
                if doc.image_path and Path(doc.image_path).exists():
                    image_vector = self.image_vectorizer.vectorize_single(doc.image_path)
                    self.db_client.upsert(
                        collection_name="image_vectors",
                        points=[PointStruct(
                            id=doc.id,
                            vector=image_vector.tolist(),
                            payload={'image_path': doc.image_path, **(doc.metadata or {})}
                        )]
                    )

                # Stats
                elapsed = time.time() - start_time
                self.stats['processed'] += 1
                self.stats['total_time'] += elapsed

                logger.info(f"Processed doc {doc.id} in {elapsed:.2f}s")
                return True

            except Exception as e:
                logger.error(f"Attempt {attempt+1} failed for doc {doc.id}: {e}")
                if attempt == retry - 1:
                    self.stats['errors'] += 1
                    return False
                time.sleep(2 ** attempt)  # Exponential backoff

        return False

    def process_batch(self, documents: List[Document], batch_size=100):
        """Traite un batch de documents"""
        logger.info(f"Processing {len(documents)} documents...")

        for i in range(0, len(documents), batch_size):
            batch = documents[i:i+batch_size]

            for doc in batch:
                self.process_document(doc)

            # Log progress
            progress = min(i + batch_size, len(documents))
            logger.info(f"Progress: {progress}/{len(documents)} ({100*progress/len(documents):.1f}%)")

        # Summary
        avg_time = self.stats['total_time'] / max(self.stats['processed'], 1)
        logger.info(f"\n=== Pipeline Summary ===")
        logger.info(f"Processed: {self.stats['processed']}")
        logger.info(f"Errors: {self.stats['errors']}")
        logger.info(f"Avg time per doc: {avg_time:.3f}s")
        logger.info(f"Throughput: {self.stats['processed']/self.stats['total_time']:.1f} docs/s")

    def search(self, query_text: Optional[str] = None,
               query_image: Optional[str] = None,
               limit=10) -> Dict:
        """Recherche unifiée texte/image"""
        results = {}

        # Recherche texte
        if query_text:
            query_vector = self.text_vectorizer.vectorize([query_text])[0]
            text_results = self.db_client.search(
                collection_name="text_vectors",
                query_vector=query_vector,
                limit=limit
            )
            results['text'] = text_results

        # Recherche image
        if query_image:
            query_vector = self.image_vectorizer.vectorize_single(query_image)
            image_results = self.db_client.search(
                collection_name="image_vectors",
                query_vector=query_vector.tolist(),
                limit=limit
            )
            results['image'] = image_results

        return results

# Utilisation
if __name__ == "__main__":
    config = {
        'text': {'model_name': 'all-MiniLM-L6-v2', 'batch_size': 32},
        'image': {'model_name': 'resnet50'},
        'vector_db': {'host': 'localhost', 'port': 6333}
    }

    pipeline = ProductionPipeline(config)
    pipeline.setup_collections()

    # Documents exemple
    documents = [
        Document(
            id=str(uuid.uuid4()),
            text="Paris est magnifique au printemps",
            image_path="./images/paris.jpg",
            metadata={'category': 'travel', 'city': 'Paris'}
        ),
        Document(
            id=str(uuid.uuid4()),
            text="La Tour Eiffel illuminée la nuit",
            image_path="./images/eiffel_night.jpg",
            metadata={'category': 'monument', 'city': 'Paris'}
        )
    ]

    # Traitement
    pipeline.process_batch(documents)

    # Recherche
    results = pipeline.search(query_text="monuments parisiens")
    print("\nRésultats:", results)

from sentence_transformers import SentenceTransformer
import torch
from tqdm import tqdm

model = SentenceTransformer('all-MiniLM-L6-v2')
model.to('cuda' if torch.cuda.is_available() else 'cpu')

# 10K documents
documents = [f"Document {i}" for i in range(10000)]

# Méthode 1 : Batch automatique (recommandé)
vectors = model.encode(
    documents,
    batch_size=32,  # Adapter selon VRAM disponible
    show_progress_bar=True,
    normalize_embeddings=True
)

# Méthode 2 : Batch manuel avec contrôle fin
def manual_batch_encode(texts, model, batch_size=32):
    all_vectors = []
    for i in tqdm(range(0, len(texts), batch_size)):
        batch = texts[i:i+batch_size]
        batch_vectors = model.encode(batch, convert_to_numpy=True)
        all_vectors.append(batch_vectors)
    return np.vstack(all_vectors)

vectors = manual_batch_encode(documents, model, batch_size=32)

import torch
from sentence_transformers import SentenceTransformer

model = SentenceTransformer('all-mpnet-base-v2')

# Vérifier nb GPUs
if torch.cuda.device_count() > 1:
    print(f"Using {torch.cuda.device_count()} GPUs")

    # Activer multi-GPU
    model = model.to('cuda')
    model._first_module().auto_model = torch.nn.DataParallel(
        model._first_module().auto_model
    )

# Encode (distribué automatiquement sur GPUs)
vectors = model.encode(documents, batch_size=64)

from multiprocessing import Pool, cpu_count
from sentence_transformers import SentenceTransformer
import numpy as np

def vectorize_chunk(args):
    """Fonction worker pour pool"""
    texts, model_name = args
    model = SentenceTransformer(model_name)
    return model.encode(texts)

def parallel_vectorize(documents, model_name='all-MiniLM-L6-v2', n_workers=None):
    if n_workers is None:
        n_workers = cpu_count() - 1

    # Diviser en chunks
    chunk_size = len(documents) // n_workers
    chunks = [documents[i:i+chunk_size] for i in range(0, len(documents), chunk_size)]

    # Pool de workers
    with Pool(n_workers) as pool:
        args = [(chunk, model_name) for chunk in chunks]
        results = pool.map(vectorize_chunk, args)

    return np.vstack(results)

# Utilisation
vectors = parallel_vectorize(documents, n_workers=8)

from sentence_transformers import SentenceTransformer
import json
from qdrant_client import QdrantClient
from qdrant_client.models import PointStruct
import uuid

model = SentenceTransformer('all-MiniLM-L6-v2')
client = QdrantClient(host='localhost', port=6333)

def stream_from_jsonl(file_path, batch_size=100):
    """Générateur streaming depuis JSONL"""
    batch = []
    with open(file_path, 'r') as f:
        for line in f:
            doc = json.loads(line)
            batch.append(doc)

            if len(batch) >= batch_size:
                yield batch
                batch = []

    if batch:  # Dernier batch
        yield batch

# Traitement streaming
for batch in stream_from_jsonl('large_dataset.jsonl', batch_size=100):
    # Vectorisation
    texts = [doc['text'] for doc in batch]
    vectors = model.encode(texts)

    # Upload immédiat vers Qdrant
    points = [
        PointStruct(
            id=str(uuid.uuid4()),
            vector=vec.tolist(),
            payload={'text': doc['text']}
        )
        for doc, vec in zip(batch, vectors)
    ]
    client.upsert(collection_name='docs', points=points)

    # Batch traité puis libéré de mémoire
    del batch, vectors, points

import gc
import torch

for i, batch in enumerate(batches):
    vectors = model.encode(batch)
    # ... traitement ...

    # Libération mémoire
    del vectors, batch

    # Garbage collection toutes les 10 itérations
    if i % 10 == 0:
        gc.collect()
        if torch.cuda.is_available():
            torch.cuda.empty_cache()

import h5py
import numpy as np

# Phase 1 : Vectorisation vers HDF5
with h5py.File('vectors.h5', 'w') as f:
    # Pré-allocation dataset
    dset = f.create_dataset('vectors', shape=(1000000, 384), dtype='float32')

    start_idx = 0
    for batch in stream_documents(batch_size=1000):
        vectors = model.encode(batch)
        end_idx = start_idx + len(vectors)
        dset[start_idx:end_idx] = vectors
        start_idx = end_idx

# Phase 2 : Upload vers base vectorielle
with h5py.File('vectors.h5', 'r') as f:
    vectors = f['vectors']
    for i in range(0, len(vectors), 1000):
        batch = vectors[i:i+1000]
        # Upload vers Qdrant/Pinecone
        client.upsert(...)

from sentence_transformers import SentenceTransformer
import torch

model = SentenceTransformer('all-MiniLM-L6-v2')
model.to('cuda')

# Activation mixed precision
model.half()  # Convertit modèle en FP16

vectors = model.encode(documents, batch_size=64)  # 2x plus rapide

# Alternative : Automatic Mixed Precision (AMP)
from torch.cuda.amp import autocast

with autocast():
    vectors = model.encode(documents)

from optimum.onnxruntime import ORTModelForFeatureExtraction
from transformers import AutoTokenizer

# Export vers ONNX (une seule fois)
model_name = "sentence-transformers/all-MiniLM-L6-v2"
model = ORTModelForFeatureExtraction.from_pretrained(
    model_name,
    export=True,
    provider="CUDAExecutionProvider"  # ou "CPUExecutionProvider"
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Inférence (1.5-2x plus rapide)
inputs = tokenizer(documents, padding=True, truncation=True, return_tensors="pt")
with torch.no_grad():
    outputs = model(**inputs)

vectors = outputs.last_hidden_state.mean(dim=1)

from optimum.onnxruntime import ORTQuantizer
from optimum.onnxruntime.configuration import AutoQuantizationConfig

# Quantization INT8
quantizer = ORTQuantizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
qconfig = AutoQuantizationConfig.avx512_vnni(is_static=False, per_channel=False)
quantizer.quantize(save_dir="./quantized_model", quantization_config=qconfig)

# Chargement modèle quantizé
model = ORTModelForFeatureExtraction.from_pretrained("./quantized_model")
# 2-3x plus rapide sur CPU

import redis
import hashlib
import json
import numpy as np
from sentence_transformers import SentenceTransformer
from qdrant_client import QdrantClient

class VectorCache:
    def __init__(self):
        self.redis_client = redis.Redis(host='localhost', port=6379, db=0)
        self.qdrant_client = QdrantClient(host='localhost', port=6333)
        self.model = SentenceTransformer('all-MiniLM-L6-v2')

    def _hash_text(self, text: str) -> str:
        """Hash texte pour clé cache"""
        return hashlib.md5(text.encode()).hexdigest()

    def get_vector(self, text: str) -> np.ndarray:
        """Récupère vecteur avec cache multi-niveaux"""
        cache_key = f"vec:{self._hash_text(text)}"

        # L1 : Redis
        cached = self.redis_client.get(cache_key)
        if cached:
            print("L1 Cache hit (Redis)")
            return np.frombuffer(cached, dtype=np.float32)

        # L2 : Qdrant
        results = self.qdrant_client.scroll(
            collection_name="vectors",
            scroll_filter={"must": [{"key": "text", "match": {"value": text}}]},
            limit=1
        )
        if results[0]:
            print("L2 Cache hit (Qdrant)")
            vector = np.array(results[0][0].vector)
            # Populate L1
            self.redis_client.setex(cache_key, 3600, vector.tobytes())
            return vector

        # L3 : Calcul + stockage
        print("Cache miss - Computing vector")
        vector = self.model.encode(text)

        # Store L1
        self.redis_client.setex(cache_key, 3600, vector.tobytes())

        # Store L2
        from qdrant_client.models import PointStruct
        import uuid
        self.qdrant_client.upsert(
            collection_name="vectors",
            points=[PointStruct(
                id=str(uuid.uuid4()),
                vector=vector.tolist(),
                payload={'text': text}
            )]
        )

        return vector

# Utilisation
cache = VectorCache()
vec1 = cache.get_vector("Paris est magnifique")  # Cache miss
vec2 = cache.get_vector("Paris est magnifique")  # L1 hit (Redis) - instantané

class VersionedVectorCache:
    MODEL_VERSION = "all-MiniLM-L6-v2-v1"  # Changer si modèle change

    def _cache_key(self, text: str) -> str:
        hash_val = hashlib.md5(text.encode()).hexdigest()
        return f"vec:{self.MODEL_VERSION}:{hash_val}"

    # Si MODEL_VERSION change, ancien cache automatiquement invalidé

import boto3
import pickle
import gzip

def save_vectors_to_s3(vectors, metadata, bucket, key):
    """Sauvegarde vecteurs compressés sur S3"""
    s3 = boto3.client('s3')

    data = {'vectors': vectors, 'metadata': metadata}
    pickled = pickle.dumps(data)
    compressed = gzip.compress(pickled, compresslevel=9)

    s3.put_object(
        Bucket=bucket,
        Key=key,
        Body=compressed,
        ContentType='application/gzip'
    )
    print(f"Saved {len(vectors)} vectors (compression: {100*(1-len(compressed)/len(pickled)):.1f}%)")

def load_vectors_from_s3(bucket, key):
    """Charge vecteurs depuis S3"""
    s3 = boto3.client('s3')
    response = s3.get_object(Bucket=bucket, Key=key)
    compressed = response['Body'].read()
    pickled = gzip.decompress(compressed)
    return pickle.loads(pickled)

# Utilisation
vectors = model.encode(documents)
save_vectors_to_s3(vectors, metadata, 'my-bucket', 'vectors/batch_001.pkl.gz')

# Vectorisation produits (texte + image)
class ProductVectorizer:
    def __init__(self):
        # Texte : description produit
        self.text_model = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2')
        # Image : photo produit
        self.image_model = CLIPModel.from_pretrained('openai/clip-vit-base-patch32')

    def vectorize_product(self, product):
        # Texte (titre + description)
        text = f"{product['title']} {product['description']}"
        text_vec = self.text_model.encode(text)  # (768,)

        # Image
        image = Image.open(product['image_path'])
        image_vec = self.image_model.encode_image(image)  # (512,)

        # Concaténation
        combined_vec = np.concatenate([text_vec, image_vec])  # (1280,)
        return combined_vec / np.linalg.norm(combined_vec)  # Normalisation

class SupportTicketSystem:
    def __init__(self):
        self.vectorizer = SentenceTransformer('all-mpnet-base-v2')
        self.qdrant = QdrantClient('localhost', port=6333)

    def index_ticket(self, ticket):
        """Indexe nouveau ticket"""
        # Concaténation titre + description + résolution
        text = f"{ticket['title']}\n{ticket['description']}\n{ticket['resolution']}"
        vector = self.vectorizer.encode(text)

        self.qdrant.upsert(
            collection_name='tickets',
            points=[PointStruct(
                id=ticket['id'],
                vector=vector.tolist(),
                payload={
                    'title': ticket['title'],
                    'category': ticket['category'],
                    'resolution_time': ticket['resolution_time'],
                    'satisfaction_score': ticket['satisfaction_score']
                }
            )]
        )

    def find_similar_tickets(self, new_ticket_description, top_k=5):
        """Trouve tickets similaires résolus"""
        query_vec = self.vectorizer.encode(new_ticket_description)

        results = self.qdrant.search(
            collection_name='tickets',
            query_vector=query_vec.tolist(),
            limit=top_k,
            query_filter={  # Seulement tickets résolus avec bonne satisfaction
                "must": [
                    {"key": "status", "match": {"value": "resolved"}},
                    {"key": "satisfaction_score", "range": {"gte": 4}}
                ]
            }
        )

        return results

    def auto_suggest_response(self, ticket_description):
        """Suggère réponse basée sur tickets similaires"""
        similar = self.find_similar_tickets(ticket_description, top_k=3)

        # Agrégation réponses
        resolutions = [hit.payload['resolution'] for hit in similar]

        # Prompt LLM pour synthèse
        prompt = f"""Tickets similaires résolus:
{chr(10).join(resolutions)}

Nouveau ticket: {ticket_description}

Suggérer une réponse:"""

        # Appel GPT-4 pour générer réponse personnalisée
        response = openai.ChatCompletion.create(
            model="gpt-4",
            messages=[{"role": "user", "content": prompt}]
        )

        return response.choices[0].message.content

import whisper
import clip
import torch
from scenedetect import VideoManager, SceneManager

class VideoVectorizer:
    def __init__(self):
        self.whisper = whisper.load_model('medium')  # Transcription audio
        self.clip = clip.load('ViT-B/32')  # Image embeddings

    def process_video(self, video_path):
        """Vectorise vidéo complète"""
        # 1. Détection de scènes
        scenes = self.detect_scenes(video_path)  # Retourne timestamps

        vectors = []
        for scene_start, scene_end in scenes:
            # 2. Extraction frame clé
            keyframe = self.extract_keyframe(video_path, scene_start)
            image_vec = self.vectorize_frame(keyframe)  # CLIP (512D)

            # 3. Transcription audio segment
            audio_segment = self.extract_audio(video_path, scene_start, scene_end)
            transcript = self.whisper.transcribe(audio_segment)['text']
            text_vec = self.vectorize_text(transcript)  # Sentence-BERT (768D)

            # 4. Fusion multi-modale
            combined = np.concatenate([image_vec, text_vec])  # (1280D)
            combined = combined / np.linalg.norm(combined)

            vectors.append({
                'vector': combined,
                'scene_start': scene_start,
                'scene_end': scene_end,
                'transcript': transcript,
                'keyframe_path': keyframe
            })

        return vectors

    def search_video_content(self, query_text):
        """Recherche sémantique dans archives"""
        query_vec = self.vectorize_text(query_text)

        # Recherche Qdrant
        results = self.qdrant.search(
            collection_name='video_scenes',
            query_vector=query_vec.tolist(),
            limit=20
        )

        # Grouper par vidéo source
        videos = {}
        for hit in results:
            video_id = hit.payload['video_id']
            if video_id not in videos:
                videos[video_id] = []
            videos[video_id].append({
                'score': hit.score,
                'timestamp': hit.payload['scene_start'],
                'transcript': hit.payload['transcript']
            })

        return videos