Implement feature X to enhance user experience and fix bug Y in module Z

- Added TP_ACE.pdf to the project.
- Introduced perf_circle.csv containing performance circle data.
- Added test_lissage.csv with test data for smoothing.
- Updated pyproject.toml to include statsmodels as a dependency.
- Modified uv.lock to reflect the addition of statsmodels and its dependencies.

.gitignore

@@ -24,4 +24,12 @@ Baudelaire_len_32.p
 
 NoticeTechnique_files
 .posit
-renv
+renv
+
+results/
+results_stage_1/
+results_stage_2/
+*.safetensors
+*.pt
+*.pth
+*.bin

M2/Advanced Machine Learning/TP2/perf_circle.csv

@@ -0,0 +1,101 @@
+x;y
+1;0
+0.997986676;0.06342392
+0.991954813;0.126592454
+0.981928697;0.189251244
+0.967948701;0.251147987
+0.950071118;0.312033446
+0.928367933;0.371662456
+0.902926538;0.429794912
+0.873849377;0.486196736
+0.841253533;0.540640817
+0.805270258;0.592907929
+0.766044443;0.64278761
+0.723734038;0.690079011
+0.678509412;0.734591709
+0.630552667;0.776146464
+0.58005691;0.814575952
+0.527225468;0.84972543
+0.472271075;0.881453363
+0.415415013;0.909631995
+0.356886222;0.93414786
+0.296920375;0.954902241
+0.235758936;0.971811568
+0.173648178;0.984807753
+0.1108382;0.993838464
+0.047581916;0.998867339
+-0.015865964;0.999874128
+-0.079249957;0.996854776
+-0.142314838;0.989821442
+-0.204806668;0.978802446
+-0.266473814;0.963842159
+-0.327067963;0.945000819
+-0.386345126;0.922354294
+-0.444066613;0.895993774
+-0.5;0.866025404
+-0.553920064;0.832569855
+-0.605609687;0.795761841
+-0.654860734;0.755749574
+-0.701474888;0.712694171
+-0.74526445;0.666769001
+-0.786053095;0.618158986
+-0.823676581;0.567059864
+-0.857983413;0.513677392
+-0.888835449;0.458226522
+-0.916108457;0.400930535
+-0.939692621;0.342020143
+-0.959492974;0.281732557
+-0.975429787;0.220310533
+-0.987438889;0.158001396
+-0.995471923;0.095056043
+-0.999496542;0.031727933
+-0.999496542;-0.031727933
+-0.995471923;-0.095056043
+-0.987438889;-0.158001396
+-0.975429787;-0.220310533
+-0.959492974;-0.281732557
+-0.939692621;-0.342020143
+-0.916108457;-0.400930535
+-0.888835449;-0.458226522
+-0.857983413;-0.513677392
+-0.823676581;-0.567059864
+-0.786053095;-0.618158986
+-0.74526445;-0.666769001
+-0.701474888;-0.712694171
+-0.654860734;-0.755749574
+-0.605609687;-0.795761841
+-0.553920064;-0.832569855
+-0.5;-0.866025404
+-0.444066613;-0.895993774
+-0.386345126;-0.922354294
+-0.327067963;-0.945000819
+-0.266473814;-0.963842159
+-0.204806668;-0.978802446
+-0.142314838;-0.989821442
+-0.079249957;-0.996854776
+-0.015865964;-0.999874128
+0.047581916;-0.998867339
+0.1108382;-0.993838464
+0.173648178;-0.984807753
+0.235758936;-0.971811568
+0.296920375;-0.954902241
+0.356886222;-0.93414786
+0.415415013;-0.909631995
+0.472271075;-0.881453363
+0.527225468;-0.84972543
+0.58005691;-0.814575952
+0.630552667;-0.776146464
+0.678509412;-0.734591709
+0.723734038;-0.690079011
+0.766044443;-0.64278761
+0.805270258;-0.592907929
+0.841253533;-0.540640817
+0.873849377;-0.486196736
+0.902926538;-0.429794912
+0.928367933;-0.371662456
+0.950071118;-0.312033446
+0.967948701;-0.251147987
+0.981928697;-0.189251244
+0.991954813;-0.126592454
+0.997986676;-0.06342392
+1;-2.45E-16

M2/Advanced Machine Learning/TP2/test_lissage.csv

@@ -0,0 +1,286 @@
+x;y
+1;12.6
+3;11.9
+5;12.8
+6;11.8
+8;12.3
+10;12.3
+12;11.7
+14;10.5
+15;10.6
+16;10.8
+20;9.7
+21;10.2
+22;9.7
+23;10.1
+25;11.8
+26;11.5
+27;11.6
+30;11.8
+31;11.8
+33;12.4
+34;13.3
+35;12.4
+39;12.9
+40;13.1
+42;13.2
+44;13.3
+46;13.2
+48;14.7
+51;13.3
+52;13.3
+55;12.5
+57;11.8
+60;11.7
+61;11.7
+63;12.2
+65;13.6
+67;12.4
+68;12.5
+70;13.2
+71;13.2
+72;13.6
+73;13.2
+77;14.4
+79;14.3
+80;14.2
+83;12.8
+84;12.8
+85;12.8
+87;12.9
+89;13.8
+91;13.2
+92;12.5
+98;13
+99;13
+100;13.4
+101;12.7
+104;12.3
+111;11.4
+112;11.1
+115;10.6
+117;10.5
+122;10.9
+123;11.1
+125;11.2
+127;11.9
+129;11
+132;10
+133;10.9
+134;10.5
+135;10.6
+138;10.4
+139;11.1
+140;11.1
+141;11.2
+144;9.9
+145;10.6
+146;9.7
+150;10.1
+151;9.9
+152;10.1
+153;10.2
+154;11.3
+155;11.4
+157;11.3
+159;11.7
+160;11.3
+162;11.4
+164;10.8
+165;10.8
+166;10.9
+168;10.8
+170;12
+171;12.4
+175;12.7
+176;12.1
+177;12.2
+179;12.3
+180;12.3
+181;12.5
+182;12.6
+183;11.8
+185;10.9
+186;10.9
+187;11.1
+188;11.1
+189;10.6
+191;9.9
+192;9.1
+193;9.6
+196;9.7
+197;10.3
+198;10.1
+200;11.7
+201;11.5
+203;11.1
+204;11.1
+205;11.6
+206;11.8
+210;11.2
+216;9.6
+217;9.1
+218;8.2
+219;7.3
+221;9
+230;9
+231;7.2
+234;7
+237;8.5
+239;8.2
+241;7.5
+242;7.2
+244;8.3
+245;8.5
+246;8.7
+250;8.8
+252;8.4
+254;8.4
+255;8.9
+258;8.3
+259;8.5
+260;9
+265;8.2
+269;8.6
+271;8
+274;8.8
+275;8.7
+276;8.6
+282;8.8
+283;9.3
+287;8.4
+290;8.6
+291;8.8
+292;9.3
+298;9.9
+301;9.6
+304;8
+306;10.6
+309;8.9
+310;8.7
+311;8.3
+313;7.6
+315;6.9
+316;6.3
+318;9.1
+321;6.8
+322;6.4
+324;6.9
+326;6.5
+331;7.5
+335;7.1
+337;7.1
+339;6.8
+341;6.6
+342;6.1
+343;6.7
+345;6.6
+346;5.7
+347;6.7
+348;7.1
+349;6.7
+351;6.7
+352;6.4
+355;6.2
+356;6.2
+357;6.4
+359;6.8
+366;6.4
+367;7.4
+368;7.4
+369;7.5
+370;7.2
+374;6.8
+378;5.8
+379;6.4
+381;5.9
+382;5.2
+386;4.7
+389;4.8
+392;4.8
+393;4.5
+394;5
+396;4.9
+399;4.5
+400;4.6
+402;4.2
+403;4.8
+406;5
+407;4.5
+411;7
+412;3.4
+413;4.1
+415;6.1
+416;5.8
+419;6.5
+420;6.4
+425;5.7
+427;5.5
+430;5.3
+431;5.8
+432;5.6
+433;6.3
+435;5.2
+438;5.4
+439;5
+443;5.3
+445;5.3
+446;5.2
+449;4.1
+450;6.9
+451;3.7
+452;3.4
+453;3.6
+455;3.5
+456;2.9
+459;2.7
+460;3.1
+462;3.7
+463;4.2
+465;4.2
+466;4
+467;3.8
+468;4
+469;3.4
+470;3.6
+475;3.7
+478;4.2
+482;3.4
+483;3.5
+485;3.1
+486;3.6
+492;5.5
+494;3.8
+496;5.5
+498;6.4
+500;5.5
+502;6.8
+503;8.2
+506;4.9
+508;5.4
+510;5.9
+514;6.4
+522;7.1
+524;7.6
+525;7
+527;6.9
+529;7.3
+533;7
+534;7.8
+536;8
+539;8.8
+540;9.1
+541;8.2
+542;8
+544;8.8
+545;9.7
+546;12
+547;6.3
+548;5.8
+556;6.3
+558;6.4
+562;7.4
+563;7.4
+566;7.2
+567;7.4
+573;7.4

M2/Clustering In Practice/compression_image.R

@@ -0,0 +1,185 @@
+# Objectifs pédagogiques
+# Comprendre la représentation matricielle d'une image.
+# Interpréter les centroïdes comme une palette de couleurs optimale (résumé).
+# Analyser le compromis entre distorsion (perte de qualité) et taux de compression.
+
+library(jpeg)
+
+# 1. Chargement de l'image
+
+img <- readJPEG("./data/PampasGrass.jpg")
+
+# Dimensions
+dims <- dim(img)
+dims
+
+# Reshaping : Transformation en matrice (Pixels x 3 canaux)
+# Chaque ligne est une observation dans R^3
+img_matrix <- matrix(img, ncol = 3)
+colnames(img_matrix) <- c("R", "G", "B")
+
+head(img_matrix)
+
+# 2. Application de l'algorithme K-means
+# Choix du nombre de couleurs (k)
+k <- 8
+
+# Application de K-means
+# On augmente iter.max car la convergence sur des milliers de pixels peut être lente
+set.seed(123)
+km_model <- kmeans(img_matrix, centers = k, iter.max = 20, nstart = 3)
+
+# Les "résumés" de l'information (les centres des clusters)
+palette_optimale <- km_model$centers
+print(palette_optimale)
+
+# 3. Reconstruction de l'image compressée
+# Associer chaque pixel à son centroïde
+img_compressed_matrix <- palette_optimale[km_model$cluster, ]
+
+# Re-transformer la matrice en Array 3D
+img_compressed <- array(img_compressed_matrix, dim = dims)
+
+# Affichage comparatif
+par(mfrow = c(1, 2), mar = c(1, 1, 1, 1))
+plot(0, 0, type='n', axes=FALSE, ann=FALSE)
+rasterImage(img, -1, -1, 1, 1)
+title("Originale (Millions de couleurs)")
+
+plot(0, 0, type='n', axes=FALSE, ann=FALSE)
+rasterImage(img_compressed, -1, -1, 1, 1)
+title(paste("Compressée (k =", k, ")"))
+
+# 4. Questions : coût de l'information (Distorsion)
+# Calculez l'erreur quadratique moyenne (MSE) entre l'image originale et 
+# l'image compressée :
+# Plus $k$ est petit, plus le résumé est ..., plus le MSE .....
+
+library(imager)
+
+
+mse_imager <- function(img1, img2) {
+  # Harmoniser dimensions (recadrage ou redimensionnement si besoin)
+  if (!all(dim(img1) == dim(img2))) {
+    # Ici, on redimensionne img2 sur la taille d'img1
+    img2 <- imresize(img2, size_x = width(img1), size_y = height(img1))
+    if (spectrum(img2) != spectrum(img1)) {
+      img2 <- grayscale(img2)  # fallback simple si nb de canaux diffère
+      img1 <- grayscale(img1)
+    }
+  }
+  # Convertir en vecteurs numériques [0,1]
+  x <- as.numeric(img1)
+  y <- as.numeric(img2)
+  mean((x - y)^2)
+}
+
+
+mse_val <- mse_imager(img, img_compressed)
+cat("MSE =", mse_val, "\n")
+
+mse_matrix <- mean((img_matrix - img_compressed_matrix)^2)
+cat("MSE =", mse_matrix, "\n")
+
+########################################################################
+
+
+
+
+
+# Règle du coude (Elbow Method)
+# tracez l'évolution de la Within-Cluster Sum of Squares (WCSS) en fonction de $k$
+# Prnde k = 2 à 32
+# A partir de quel $k$ le gain visuel devient-il négligeable pour l'œil humain ?
+
+
+
+# X : matrice/df n x d
+# ks : valeurs de k à tester (par défaut 1:10)
+elbow_wss <- function(X, ks = 2:32, nstart = 10, scale_data = FALSE) {
+  X <- as.matrix(X)
+  if (scale_data) {
+    X <- scale(X)
+  }
+  wss <- numeric(length(ks))
+  
+  # Cas k = 1 : WSS = TSS (variance totale)
+  total_ss <- sum(scale(X, scale = FALSE)^2)  # TSS
+  for (i in seq_along(ks)) {
+    k <- ks[i]
+    cat(" k =", k, "\n")
+    if (k == 1) {
+      wss[i] <- total_ss
+    } else {
+      set.seed(123)  # reproductible
+      km <- kmeans(X, centers = k, nstart = nstart, iter.max = 100)
+      wss[i] <- km$tot.withinss
+    }
+  }
+  
+  plot(ks, wss, type = "b", pch = 19, xlab = "Nombre de clusters (k)",
+       ylab = "Inertie intra-classe (WSS)",
+       main = "Méthode du coude (k-means)")
+  grid()
+#  invisible(data.frame(k = ks, WSS = wss))
+}
+
+# Exemple d'utilisation :
+ res <- elbow_wss(img_compressed, ks = 2:32, nstart = 20, scale_data = FALSE)
+
+###############################################################################
+ 
+ elbow_wss_safe <- function(X, ks = 2:32, nstart = 20, scale_data = FALSE, seed = 123) {
+   X <- as.matrix(X)
+   if (scale_data) X <- scale(X)
+   set.seed(seed)
+   
+   # Nombre de lignes distinctes
+   n_unique <- nrow(unique(X))
+   if (n_unique < 2) stop("Moins de 2 points distincts : k-means n'a pas de sens.")
+   
+   # Tronquer ks si nécessaire
+   ks <- ks[ks <= n_unique]
+   if (length(ks) == 0) stop("Tous les k demandés dépassent le nombre de points distincts.")
+   
+   wss <- numeric(length(ks))
+   # TSS (k = 1)
+   total_ss <- sum(scale(X, scale = FALSE)^2)
+   
+   for (i in seq_along(ks)) {
+     k <- ks[i]
+     cat(" k =", k, "\n")
+     if (k == 1) {
+       wss[i] <- total_ss
+     } else {
+       km <- kmeans(X, centers = k, nstart = nstart, iter.max = 100)
+       wss[i] <- km$tot.withinss
+     }
+   }
+   
+   plot(ks, wss, type = "b", pch = 19, xlab = "Nombre de clusters (k)",
+        ylab = "Inertie intra-classe (WSS)", main = "Méthode du coude (k-means)")
+   axis(1, at = ks)
+   grid()
+#   invisible(data.frame(k = ks, WSS = wss))
+ }
+ 
+ # Exemple :
+  res <- elbow_wss_safe(img_compressed, ks = 2:32, nstart = 20)
+ 
+
+
+
+# Taille de stockage
+# Ouvrir un fichier JPG
+jpeg("./data/image_compressed.jpg")
+
+# Afficher l'image compressée dans le fichier
+plot(0, 0, type='n', axes=FALSE, ann=FALSE)
+rasterImage(img_compressed, -1, -1, 1, 1)
+
+info <- file.info("./data/PampasGrass.jpg")
+(taille_octets_reelle <- info$size/1024)
+
+info <- file.info("./data/image_compressed.jpg")
+(taille_octets_compresse <- info$size/1024)

M2/Clustering In Practice/compression_image_poissons.R

@@ -0,0 +1,105 @@
+# Objectifs pédagogiques
+# Comprendre la représentation matricielle d'une image.
+# Interpréter les centroïdes comme une palette de couleurs optimale (résumé).
+# Analyser le compromis entre distorsion (perte de qualité) et taux de compression.
+
+library(jpeg)
+
+# 1. Chargement de l'image
+
+
+img <- readJPEG("./data/Guppy 2.jpeg")
+
+# Dimensions
+dims <- dim(img)
+dims
+
+# Reshaping : Transformation en matrice (Pixels x 3 canaux)
+# Chaque ligne est une observation dans R^3
+img_matrix <- matrix(img, ncol = 3)
+colnames(img_matrix) <- c("R", "G", "B")
+
+head(img_matrix)
+
+# 2. Application de l'algorithme K-means
+# Choix du nombre de couleurs (k)
+k <- 10
+
+# Application de K-means
+# On augmente iter.max car la convergence sur des milliers de pixels peut être lente
+set.seed(123)
+km_model <- kmeans(img_matrix, centers = k, iter.max = 20, nstart = 3)
+
+# Les "résumés" de l'information (les centres des clusters)
+palette_optimale <- km_model$centers
+print(palette_optimale)
+
+# 3. Reconstruction de l'image compressée
+# Associer chaque pixel à son centroïde
+img_compressed_matrix <- palette_optimale[km_model$cluster, ]
+
+# Re-transformer la matrice en Array 3D
+img_compressed <- array(img_compressed_matrix, dim = dims)
+
+# Affichage comparatif
+par(mfrow = c(1, 2), mar = c(1, 1, 1, 1))
+plot(0, 0, type='n', axes=FALSE, ann=FALSE)
+rasterImage(img, -1, -1, 1, 1)
+title("Originale (Millions de couleurs)")
+
+plot(0, 0, type='n', axes=FALSE, ann=FALSE)
+rasterImage(img_compressed, -1, -1, 1, 1)
+title(paste("Compressée (k =", k, ")"))
+
+# 4. Questions : coût de l'information (Distorsion)
+# Calculez l'erreur quadratique moyenne (MSE) entre l'image originale et 
+# l'image compressée :
+# Plus $k$ est petit, plus le résumé est ..., plus le MSE .....
+
+# Règle du coude (Elbow Method)
+# tracez l'évolution de la Within-Cluster Sum of Squares (WCSS) en fonction de $k$
+# Prnde k = 2 à 32
+# A partir de quel $k$ le gain visuel devient-il négligeable pour l'œil humain ?
+
+# Taille de stockage
+# Ouvrir un fichier JPG
+jpeg("./data/image_compressed.jpg")
+
+# Afficher l'image compressée dans le fichier
+plot(0, 0, type='n', axes=FALSE, ann=FALSE)
+rasterImage(img_compressed, -1, -1, 1, 1)
+
+info <- file.info("./data/Guppy 2.jpeg")
+(taille_octets_reelle <- info$size/1024)
+
+info <- file.info("./data/image_compressed.jpg")
+(taille_octets_compresse <- info$size/1024)
+
+
+library(colordistance)
+
+repertoire <- "poissons"
+
+clusters <- colordistance::getHistList(repertoire, lower = NULL, upper = NULL)
+names(clusters)
+
+kmeans_fits <- getKMeansList(repertoire, bins = 3, plotting = TRUE)
+
+centroids_list <- extractClusters(kmeans_fits, ordering = TRUE)
+
+emd_distance_matrix <- getColorDistanceMatrix(centroids_list, method = "color.dist", ordering = TRUE)
+
+colordistance::imageClusterPipeline(repertoire, cluster.method = "hist")
+
+clusters
+
+
+
+
+
+
+
+
+
+
+

M2/Clustering In Practice/data/DataModel.csv

@@ -0,0 +1,553 @@
+"Year","Category","Region","Dist_EQ","PCT_CHARTER","PCT_NETWORK","PCT_REGIONAL","PCT_LCC"
+2002,"NBI","AFRMY",1016.77834099741,0.0597130903899918,0.622493232142152,0.267124620948855,0.0506690565190006
+2002,"NBI","AMCS",1048.40176202633,0,0.600042338293917,0.377530799113856,0.0224268625922272
+2002,"NBI","AMN",1409.85814009492,0.0173629970840251,0.590396464468702,0.173805680475439,0.218434857971835
+2002,"NBI","ASIAUS",731.613261373554,0,0.469682218115679,0.466558289304771,0.0637594925795506
+2002,"NBI","CEI",2021.61669687385,0.101248783609204,0.33592314644564,0.353537170585854,0.209290899359302
+2002,"NBI","CHN",884.36087443162,0,0.541909244621729,0.423698368369688,0.0343923870085828
+2002,"NBI","EUR",912.423381449187,0.133168395537474,0.531242916846183,0.199587502830158,0.136001184786185
+2002,"NBI","SIN",956.879704507048,0.0219844929189212,0.878033445909996,0.0999820611710825,0
+2002,"NBL","AFRMY",1870.45081950017,0.000605076293025976,0.669678363444417,0.329716560262557,0
+2002,"NBL","AMCS",1824.96151047405,0,0.809693945741259,0.0883583183642831,0.101947735894458
+2002,"NBL","AMN",1943.97876631639,0.051832775219223,0.620556018720925,0.140872307705065,0.186738898354787
+2002,"NBL","ASIAUS",765.889004994681,0,0.72582587884237,0.222936510506108,0.0512376106515214
+2002,"NBL","CEI",4013.47097243836,0.00884521845500873,0.0328186621150331,0.90034431408971,0.0579918053402481
+2002,"NBL","CHN",1144.23746822827,0,0.452582301987532,0.504259544661241,0.0431581533512276
+2002,"NBL","EUR",1234.90303545659,0.364607796353512,0.57342178096109,0.0530094145514347,0.00896100813396302
+2002,"NBL","SIN",1735.11180609851,0,0,1,0
+2002,"NBS","AFRMY",896.194534112975,0.0307407102817188,0.573908924174224,0.394921119969798,0.000429245574259246
+2002,"NBS","AMCS",747.174211603206,0,0.529624167602986,0.412755321767577,0.0576205106294371
+2002,"NBS","AMN",988.093839351939,0.0013652968487858,0.398303810665043,0.193870598522295,0.406460293963877
+2002,"NBS","ASIAUS",742.441286377743,0,0.35286235135051,0.51736909093461,0.12976855771488
+2002,"NBS","CEI",1301.34576825985,0.157995862718028,0,0.842004137281972,0
+2002,"NBS","CHN",901.974419592352,0,0.737234338617439,0.262765661382561,0
+2002,"NBS","EUR",781.859523179824,0.034147346899082,0.419150058586784,0.289664731496763,0.257037863017371
+2002,"NBS","SIN",1093.25099049704,0,0,1,0
+2003,"NBI","AFRMY",1051.83276929881,0.0703904417993861,0.58586464219364,0.297699437447877,0.0460454785590967
+2003,"NBI","AMCS",1046.0439259395,0.000159622611484496,0.580645331548867,0.381263370019119,0.0379316758205296
+2003,"NBI","AMN",1498.83664670975,0.0319213107098393,0.556433821677094,0.166345733914352,0.245299133698715
+2003,"NBI","ASIAUS",826.321630659026,2.87414444591047e-06,0.439523953566766,0.536791058278473,0.0236821140103144
+2003,"NBI","CEI",1949.75420550375,0.0575330852873459,0.298075010313518,0.434867435136563,0.209524469262573
+2003,"NBI","CHN",909.696158854379,0,0.632284602130837,0.346134092993799,0.0215813048753633
+2003,"NBI","EUR",955.788965857104,0.132691421953644,0.522631131213971,0.173206925976766,0.171470520855619
+2003,"NBI","SIN",974.713460537252,0.0158550280722602,0.808232832813693,0.171151971208312,0.00476016790573452
+2003,"NBL","AFRMY",1751.82883955615,0.0314011459979526,0.569588576896012,0.399010277106036,0
+2003,"NBL","AMCS",1822.559822083,0,0.762943162951321,0.128930594625762,0.108126242422917
+2003,"NBL","AMN",2026.0997925162,0.0644717204234929,0.598088557608768,0.139921778028566,0.197517943939173
+2003,"NBL","ASIAUS",1328.5958565509,0,0.64457537018407,0.20320958073791,0.152215049078021
+2003,"NBL","CEI",3723.64309407328,0.00870866487367566,0.00386223143758493,0.925957583814866,0.0614715198738737
+2003,"NBL","CHN",1312.91071496536,0,0.58368766901228,0.367520784078573,0.0487915469091474
+2003,"NBL","EUR",1215.06501553753,0.302037229834171,0.579633562109029,0.113156292947753,0.00517291510904663
+2003,"NBL","SIN",1549.69043928804,0,0.207197663467606,0.792802336532394,0
+2003,"NBS","AFRMY",1020.26844457009,0.0592471603459175,0.527859936259316,0.412445373213889,0.000447530180877296
+2003,"NBS","AMCS",812.867449227247,0,0.537901975918196,0.358837838554683,0.103260185527121
+2003,"NBS","AMN",1034.07827897107,0.00191780575189369,0.374634584757492,0.178652321458941,0.444795288031673
+2003,"NBS","ASIAUS",819.277607285573,0.00210191710870563,0.279343639618056,0.346537536703958,0.372016906569281
+2003,"NBS","CEI",1370.90327513558,0.201685678688886,0.0448981117980321,0.753416209513082,0
+2003,"NBS","CHN",947.421571182859,0,0.748828864793134,0.251171135206866,0
+2003,"NBS","EUR",797.83472170478,0.052057705585553,0.420549111447807,0.308357854284043,0.219035328682597
+2003,"NBS","SIN",1178.36392294334,0,0,0.924568394630158,0.0754316053698418
+2004,"NBI","AFRMY",1118.84228834104,0.0719015260194344,0.586382396634064,0.284417536153788,0.057298541192714
+2004,"NBI","AMCS",1066.33349071331,0,0.557795356455719,0.398933257242986,0.0432713863012945
+2004,"NBI","AMN",1559.76351030533,0.0371168132172758,0.457589075882335,0.167304100632898,0.337990010267491
+2004,"NBI","ASIAUS",895.061401625613,0.00351434191290419,0.407998279703067,0.578451861928994,0.0100355164550348
+2004,"NBI","CEI",1895.37246841514,0.0621895397647724,0.223517807028265,0.495640633683318,0.218652019523644
+2004,"NBI","CHN",1000.4594665654,0,0.600163096684699,0.374120748057107,0.0257161552581933
+2004,"NBI","EUR",1010.59338718854,0.132671487148077,0.497420746298331,0.152718867182171,0.21718889937142
+2004,"NBI","SIN",979.908245588873,0.000569529127851526,0.77049725220613,0.228933218666019,0
+2004,"NBL","AFRMY",1854.09068466113,0.0230344008837983,0.530715015293745,0.446250583822457,0
+2004,"NBL","AMCS",1735.4292061985,0,0.669305287395437,0.215266978003438,0.115427734601125
+2004,"NBL","AMN",2079.46013496378,0.0615241629505311,0.445398422407843,0.139054382607996,0.35402303203363
+2004,"NBL","ASIAUS",1169.74074607907,0.057163394637303,0.74791876942464,0.101251746052136,0.0936660898859209
+2004,"NBL","CEI",3414.37852598339,0.0975838497409029,0.111023915646246,0.728306869453293,0.0630853651595581
+2004,"NBL","CHN",1261.00651246869,0,0.609679750621362,0.330338461372172,0.0599817880064659
+2004,"NBL","EUR",1317.01591035747,0.250405851122713,0.623290601334873,0.101772769455369,0.0245307780870447
+2004,"NBL","SIN",1444.11232276274,0,0.419750948965739,0.580249051034261,0
+2004,"NBS","AFRMY",975.100903644071,0.0677497176473869,0.462821740223249,0.468351395459184,0.00107714667018013
+2004,"NBS","AMCS",827.996945667859,0,0.526184470650645,0.35735758077773,0.116457948571625
+2004,"NBS","AMN",1072.26228809395,0.00195151069157886,0.219300737137274,0.179046918234192,0.599700833936955
+2004,"NBS","ASIAUS",876.477227814581,0.00155311460801366,0.179458912302523,0.22482693354778,0.594161039541684
+2004,"NBS","CEI",1334.90751418335,0.216356673704201,0.313838369662649,0.468739021445937,0.00106593518721326
+2004,"NBS","CHN",956.848300480629,0,0.820224553164155,0.179516211481983,0.000259235353862591
+2004,"NBS","EUR",827.75461937925,0.0831298345762109,0.357945045100817,0.249001050066969,0.309924070256003
+2004,"NBS","SIN",1138.44344746954,0,0,1,0
+2005,"NBI","AFRMY",1144.6473085694,0.0664152811711306,0.539696049145382,0.337990821845681,0.0558978478378065
+2005,"NBI","AMCS",1066.95770202064,0,0.565873185288841,0.388196571002098,0.0459302437090615
+2005,"NBI","AMN",1547.31279759207,0.0227526536617333,0.452963215927672,0.167578737439385,0.35670539297121
+2005,"NBI","ASIAUS",902.205693075095,0.00417293984616082,0.352498092307051,0.569070563335918,0.0742584045108701
+2005,"NBI","CEI",1873.5506127118,0.0702844042456643,0.206525859735807,0.511657378206404,0.211532357812125
+2005,"NBI","CHN",1035.35139870713,0,0.630046071741499,0.343781662121277,0.0261722661372242
+2005,"NBI","EUR",1018.90251261652,0.128046913616995,0.529984291873972,0.121777702743246,0.220191091765786
+2005,"NBI","SIN",1023.06687094566,0,0.75275156733831,0.24724843266169,0
+2005,"NBL","AFRMY",1810.02105488293,0.0371926827644837,0.539387373962615,0.423419943272902,0
+2005,"NBL","AMCS",1726.75732585068,0,0.711059249437188,0.188230549920793,0.100710200642019
+2005,"NBL","AMN",2109.24405792989,0.0391014466679743,0.458734886225285,0.136562585217016,0.365601081889725
+2005,"NBL","ASIAUS",1037.04239942446,0.100698402680429,0.808376121156836,0.0185210151369094,0.0724044610258255
+2005,"NBL","CEI",3189.2428547016,0.171177549768552,0.20736132642256,0.57822815188997,0.0432329719189168
+2005,"NBL","CHN",1125.08281549542,0,0.555712767270187,0.389918708121961,0.0543685246078521
+2005,"NBL","EUR",1236.54730270984,0.283755233670644,0.631980022925384,0.0728362110640691,0.0114285323399037
+2005,"NBL","SIN",1474.32248032714,0,0.391378024983744,0.608621975016256,0
+2005,"NBS","AFRMY",944.057666641918,0.0868315093655335,0.413495892183916,0.456665085205021,0.0430075132455295
+2005,"NBS","AMCS",837.29685252464,0,0.552217955745499,0.295268249310276,0.152513794944226
+2005,"NBS","AMN",1102.98776605576,0.0050067486763006,0.21064297147412,0.163538466559019,0.62081181329056
+2005,"NBS","ASIAUS",880.594368505296,0,0.158548615316017,0.176302779128767,0.665148605555216
+2005,"NBS","CEI",1353.6195416818,0.232640433020087,0.273659282575309,0.422138212886689,0.0715620715179144
+2005,"NBS","CHN",959.782513779051,0,0.735002611782261,0.26485492276103,0.000142465456708642
+2005,"NBS","EUR",861.891616310017,0.084877102823163,0.301965006454799,0.163169501287278,0.449988389434761
+2005,"NBS","SIN",920.386833700046,0,0.0119700260861467,0.988029973913853,0
+2006,"NBI","AFRMY",1138.04732765766,0.0800975799238458,0.506326179207697,0.310658120147633,0.102918120720824
+2006,"NBI","AMCS",1031.34035186997,0,0.594644733989955,0.33380888015396,0.0715463858560854
+2006,"NBI","AMN",1587.15047211395,0.0144793536243242,0.493988590006146,0.129792047209454,0.361740009160076
+2006,"NBI","ASIAUS",942.019400916568,0.00668256126305651,0.285334186757348,0.498176437003973,0.209806814975622
+2006,"NBI","CEI",1902.06195567292,0.087699412848399,0.228088234173566,0.50140820203356,0.182804150944475
+2006,"NBI","CHN",1062.10188868693,0,0.605711891013932,0.368696072192362,0.0255920367937056
+2006,"NBI","EUR",1019.00908136861,0.125108192597397,0.483604311510742,0.138424836751621,0.252862659140239
+2006,"NBI","SIN",1049.04800558863,0,0.650299991727958,0.274523356683468,0.0751766515885743
+2006,"NBL","AFRMY",1899.32443336587,0.0135018297452449,0.5547227084213,0.431775461833455,0
+2006,"NBL","AMCS",1939.20935088796,0,0.54565098242017,0.328285859772465,0.126063157807365
+2006,"NBL","AMN",2187.67128026574,0.0161667752194186,0.57602842212399,0.03853815300604,0.369266649650551
+2006,"NBL","ASIAUS",838.495611230166,0.0245901590890957,0.950980745058982,0.0159268006244832,0.00850229522743875
+2006,"NBL","CEI",3023.53223923825,0.146023985007469,0.146985332154633,0.698296864512271,0.00869381832562711
+2006,"NBL","CHN",1134.28503381506,0,0.591546525322623,0.358284579807739,0.0501688948696376
+2006,"NBL","EUR",1310.43452032375,0.288605861840223,0.633770439586171,0.0579149345013532,0.0197087640722526
+2006,"NBL","SIN",1510.47536969813,0,0.450772203812408,0.549227796187592,0
+2006,"NBS","AFRMY",974.478817897077,0.0756597750364558,0.471440795514074,0.41963457244113,0.0332648570083407
+2006,"NBS","AMCS",858.268397159176,0,0.514976037718629,0.270613590784659,0.214410371496713
+2006,"NBS","AMN",1114.73769830693,0.00536186599671175,0.239422726677631,0.0950835749915738,0.660131832334083
+2006,"NBS","ASIAUS",873.220374167129,0,0.156823114834569,0.203876419644402,0.639300465521029
+2006,"NBS","CEI",1361.29909941249,0.170233594203652,0.187328238468591,0.491728192715042,0.150709974612716
+2006,"NBS","CHN",943.364937862824,0.0059309273700467,0.659226247821932,0.334842824808022,0
+2006,"NBS","EUR",873.34320313538,0.0867024921103081,0.262601342649803,0.187582150187531,0.463114015052359
+2006,"NBS","SIN",904.652496449135,0,0.849189109762939,0.150810890237061,0
+2007,"NBI","AFRMY",1203.54665874132,0.0585926657002463,0.509451122536068,0.271768575067512,0.160187636696174
+2007,"NBI","AMCS",1096.53436865042,0.00048342238509541,0.638132705020092,0.273647419438365,0.0877364531564478
+2007,"NBI","AMN",1614.57200208311,0.0144682302797693,0.629105237870985,0.122745188325937,0.233681343523309
+2007,"NBI","ASIAUS",988.986878274016,0.00817657984201907,0.343094041205826,0.37579422259989,0.272935156352265
+2007,"NBI","CEI",1843.83106753159,0.0807721344260225,0.262994945859939,0.538231385810682,0.118001533903357
+2007,"NBI","CHN",1087.72227782614,0.00871684618403742,0.596596783347116,0.363069168162567,0.0316172023062802
+2007,"NBI","EUR",1031.75960706547,0.136786494362055,0.410593149305638,0.143636996694623,0.308983359637684
+2007,"NBI","SIN",1026.44437882514,0,0.525161979414646,0.30293809998151,0.171899920603844
+2007,"NBL","AFRMY",1942.85550048815,0.00218389166757012,0.588470557597048,0.40929720543993,4.8345295451351e-05
+2007,"NBL","AMCS",1752.72021549169,0,0.361097899948369,0.478313640602461,0.16058845944917
+2007,"NBL","AMN",2193.13480849429,0.0149828113663743,0.78509866900973,0.0404014546028906,0.159517065021005
+2007,"NBL","ASIAUS",989.889932809892,0.0162271509217324,0.832187530626005,0.149945562183088,0.0016397562691749
+2007,"NBL","CEI",2909.34626217913,0.000850322363912121,0.277571816801808,0.72157786083428,0
+2007,"NBL","CHN",1169.68265841665,0,0.604437226859891,0.35048544113896,0.0450773320011491
+2007,"NBL","EUR",1336.48548759046,0.456332670589603,0.399129698484048,0.0511075391287189,0.0934300917976305
+2007,"NBL","SIN",1333.1696324163,0,0.816853994648282,0.183146005351718,0
+2007,"NBS","AFRMY",1003.75236095351,0.0651568400256861,0.432675433252729,0.45687060301457,0.0452971237070148
+2007,"NBS","AMCS",882.432144143286,0.00911187527644252,0.337419271740532,0.296353005902383,0.357115847080643
+2007,"NBS","AMN",1131.15899236726,0.00511992804128777,0.228541472390303,0.0795260829088163,0.686812516659593
+2007,"NBS","ASIAUS",851.405008344783,0,0.147859459329913,0.223130936448259,0.629009604221828
+2007,"NBS","CEI",1433.62733081037,0.0544880297398635,0.134230879319672,0.561145906269914,0.25013518467055
+2007,"NBS","CHN",958.820786614708,0.00201519673403784,0.672948683552199,0.325036119713764,0
+2007,"NBS","EUR",893.382551697775,0.0922905364833857,0.259615999569008,0.167117411829207,0.480976052118399
+2007,"NBS","SIN",861.497596814846,0,0.936102212290443,0.0638977877095566,0
+2008,"NBI","AFRMY",1313.54306329234,0.0516667778398931,0.547061472206402,0.235465465904931,0.165806284048774
+2008,"NBI","AMCS",1114.49127999271,0,0.575627146383918,0.226964472357331,0.197408381258751
+2008,"NBI","AMN",1629.01673594902,0.00620271978000414,0.664077881059971,0.0841953246826147,0.24552407447741
+2008,"NBI","ASIAUS",994.809090652737,0.000987946700090422,0.313624979246642,0.340936070149694,0.344451003903573
+2008,"NBI","CEI",1854.36334350641,0.0876837604099257,0.255835891839049,0.534707079924424,0.121773267826601
+2008,"NBI","CHN",1100.30413978341,0.0134513573271539,0.569656014383211,0.376550959295235,0.0403416689944007
+2008,"NBI","EUR",1043.54112546693,0.129774833686005,0.375025039275457,0.132825311109915,0.362374815928622
+2008,"NBI","SIN",1143.8008347137,0.00430039812644039,0.469988951992067,0.296132058994066,0.229578590887426
+2008,"NBL","AFRMY",1886.58991315224,0.00165812898256478,0.605155510062856,0.381504039564207,0.0116823213903723
+2008,"NBL","AMCS",1700.82158728223,0.0205919955785117,0.368989507434037,0.459661339231761,0.15075715775569
+2008,"NBL","AMN",2260.87332905502,0.00726989606142494,0.817317864743296,0.022247548087343,0.153164691107936
+2008,"NBL","ASIAUS",986.330466920578,0,0.609492711499366,0.345369469935553,0.0451378185650805
+2008,"NBL","CEI",2840.84259443527,0.00813188489772224,0.322413368793628,0.66945474630865,0
+2008,"NBL","CHN",1206.92511347414,0,0.727578560849299,0.229183584378653,0.0432378547720479
+2008,"NBL","EUR",1343.21561823585,0.448323700509502,0.392585120773226,0.0642592864143586,0.094831892302913
+2008,"NBL","SIN",1272.72578138311,0,0.877971926433105,0.122028073566895,0
+2008,"NBS","AFRMY",1005.64467742537,0.0762956621718267,0.453014830546209,0.400966646404061,0.0697228608779029
+2008,"NBS","AMCS",905.27681512962,0,0.415212580447228,0.328715964602552,0.25607145495022
+2008,"NBS","AMN",1147.49513867048,0.00358616306061373,0.236369026638581,0.0389015292315772,0.721143281069228
+2008,"NBS","ASIAUS",861.490624905805,0,0.147563247669611,0.636912321783231,0.215524430547158
+2008,"NBS","CEI",1429.536431327,0.0419663013467043,0.169225514491592,0.538045111746165,0.250763072415538
+2008,"NBS","CHN",998.620779678658,0,0.645436327900737,0.354563672099262,0
+2008,"NBS","EUR",913.83679925806,0.0792022575619804,0.274295148563102,0.167430830173512,0.479071763701405
+2008,"NBS","SIN",842.763596584005,0,0.893037698780132,0.106962301219868,0
+2009,"NBI","AFRMY",1395.6180557787,0.0400077144293168,0.563645886024902,0.169354435288079,0.226991964257702
+2009,"NBI","AMCS",1134.15923696427,0,0.549034539417961,0.194657314544938,0.256308146037101
+2009,"NBI","AMN",1636.14445625878,0.00306727606950699,0.660795491599832,0.0820724467980983,0.254064785532563
+2009,"NBI","ASIAUS",1007.76366536438,0.00160487039716765,0.295328454196822,0.311082796977006,0.391983878429004
+2009,"NBI","CEI",1831.7440983442,0.11200343505728,0.308555459025905,0.462291361915983,0.117149744000833
+2009,"NBI","CHN",1089.80047248912,0.0130309603361804,0.528815563203569,0.418075170054611,0.0400783064056394
+2009,"NBI","EUR",1077.4793370156,0.101058057927184,0.377478238815063,0.116074573263744,0.40538912999401
+2009,"NBI","SIN",1198.76062615162,0.000547586998631722,0.49611905510428,0.21785225257422,0.285481105322868
+2009,"NBL","AFRMY",1737.999604481,0.0024356014389473,0.667761381959535,0.311703744755316,0.0180992718462014
+2009,"NBL","AMCS",1786.30343201811,0,0.533278662185533,0.466721337814467,0
+2009,"NBL","AMN",2256.85781416248,0.000293962531991564,0.822998417014824,0.0192566507782951,0.157450969674889
+2009,"NBL","ASIAUS",975.288797642984,0,0.49502523471116,0.402058268815426,0.102916496473414
+2009,"NBL","CEI",2700.04791763294,0.016265741755581,0.428579284500524,0.555154973743895,0
+2009,"NBL","CHN",1312.51344831019,0,0.779308474977018,0.187962133444308,0.0327293915786742
+2009,"NBL","EUR",1332.00055759885,0.271301387749559,0.539690398469936,0.0930775018965338,0.0959307118839708
+2009,"NBL","SIN",1236.87637021478,0,0.87543498003085,0.12456501996915,0
+2009,"NBS","AFRMY",1015.19782818828,0.067724090202741,0.512349877055858,0.366186892953176,0.0537391397882253
+2009,"NBS","AMCS",922.339471792621,0,0.463090324498557,0.222086500275168,0.314823175226275
+2009,"NBS","AMN",1150.42444420275,0.0023638485091488,0.220153125835233,0.0195277289886652,0.757955296666953
+2009,"NBS","ASIAUS",862.341398000546,0,0.147486620243995,0.662139958344752,0.190373421411253
+2009,"NBS","CEI",1508.18157566757,0.0244837351288422,0.121376641024494,0.603876667548701,0.250262956297963
+2009,"NBS","CHN",1006.60539874558,0,0.624859405991935,0.356141637396337,0.0189989566117288
+2009,"NBS","EUR",912.597681541622,0.0597885210246579,0.300312732832543,0.152752448564238,0.48714629757856
+2009,"NBS","SIN",941.281238519887,0,0.784759223988131,0.214067828282338,0.00117294772953116
+2010,"NBI","AFRMY",1406.43296479054,0.0449048043681947,0.535051098553474,0.167201994631819,0.252842102446512
+2010,"NBI","AMCS",1134.83544343317,0,0.545390107797884,0.170182358466128,0.284427533735988
+2010,"NBI","AMN",1662.42359864004,0.00496061847067828,0.728816672217152,0.0769249813229236,0.189297727989246
+2010,"NBI","ASIAUS",1034.44869437252,0.00269041190310057,0.275544806830541,0.293535988808319,0.428228792458039
+2010,"NBI","CEI",1885.2361359789,0.129814317275619,0.218206627002135,0.490502210632826,0.161476845089419
+2010,"NBI","CHN",1111.75193991616,0.0147557384148718,0.498562398808354,0.441190995021097,0.0454908677556766
+2010,"NBI","EUR",1126.43528438371,0.0831563310136458,0.350644657574872,0.0893713436290118,0.476827667782471
+2010,"NBI","SIN",1232.04688847674,0,0.392683619459063,0.267490147161415,0.339826233379522
+2010,"NBL","AFRMY",1748.96251256379,0.00515544186776357,0.638631320076687,0.290611875262191,0.0656013627933582
+2010,"NBL","AMCS",1817.25210460535,0,0.569252188602098,0.430747811397902,0
+2010,"NBL","AMN",2293.48169537631,2.51886061967639e-05,0.977088844136898,0.0183975145457622,0.00448845271114271
+2010,"NBL","ASIAUS",979.060328830998,0,0.395449156088465,0.504781657452823,0.0997691864587119
+2010,"NBL","CEI",2534.23593350032,0.0364027800357403,0.447826169790593,0.515771050173667,0
+2010,"NBL","CHN",1285.01096331124,0,0.796704067562598,0.175527073808743,0.027768858628659
+2010,"NBL","EUR",1343.19436919399,0.187165060525048,0.576851827824477,0.111657707931873,0.124325403718603
+2010,"NBL","SIN",1341.1531012091,0,0.885615939288984,0.114384060711016,0
+2010,"NBS","AFRMY",1046.98052361658,0.0634349615944977,0.49049751068997,0.394055886461295,0.0520116412542372
+2010,"NBS","AMCS",917.446287524619,0,0.41819603371846,0.293179215802351,0.288624750479189
+2010,"NBS","AMN",1177.1913099698,0.00159825385375051,0.265352606783627,0.0181746340843401,0.714874505278283
+2010,"NBS","ASIAUS",865.473638725562,0,0.126644446963058,0.650364976235155,0.222990576801786
+2010,"NBS","CEI",1488.98639545524,0.017966577903667,0.115817629679639,0.644508930958306,0.221706861458388
+2010,"NBS","CHN",1034.68559085897,0.00130413828754895,0.639212948063118,0.333046610538162,0.0264363031111704
+2010,"NBS","EUR",911.023573481606,0.048294969718647,0.33208352318159,0.138074424882399,0.481547082217364
+2010,"NBS","SIN",960.523408860522,0,0.751895625744033,0.246443344797988,0.00166102945797802
+2011,"NBI","AFRMY",1424.51737732262,0.0351514254809325,0.55005406796654,0.168007335828231,0.246787170724297
+2011,"NBI","AMCS",1126.18376406221,0.000734612055461518,0.530754016639782,0.158005648584995,0.310505722719762
+2011,"NBI","AMN",1686.05998924036,0.0104602579483087,0.707861702176985,0.0778357048666881,0.203842335008018
+2011,"NBI","ASIAUS",1073.91369191156,0.00244862169795106,0.255719918276249,0.274977502095967,0.466853957929832
+2011,"NBI","CEI",1979.93800292899,0.137775539787413,0.23628000819228,0.528596890430094,0.0973475615902134
+2011,"NBI","CHN",1142.44620461285,0.0159717505033257,0.467328533995739,0.464062066137272,0.0526376493636626
+2011,"NBI","EUR",1187.57140666468,0.0880275242535338,0.328394484056784,0.0786915934967489,0.504886398192933
+2011,"NBI","SIN",1199.41243741536,0,0.39764156675056,0.211455746673591,0.390902686575849
+2011,"NBL","AFRMY",1739.6007920479,0.0040777823489101,0.732139231352923,0.253912764965224,0.00987022133294287
+2011,"NBL","AMCS",1872.53328207283,0,0.623483034203575,0.376516965796425,0
+2011,"NBL","AMN",2293.50566362053,0.00120843065221345,0.975075343474976,0.0195908168289434,0.00412540904386668
+2011,"NBL","ASIAUS",957.459125162572,0,0.3981324625055,0.522811105254002,0.0790564322404976
+2011,"NBL","CEI",2444.1952488186,0,0.369093409930154,0.630906590069846,0
+2011,"NBL","CHN",1288.51124331818,0,0.83484995943391,0.139045400403866,0.0261046401622243
+2011,"NBL","EUR",1620.02575901213,0.179912556375419,0.590261535633465,0.109784206390408,0.120041701600709
+2011,"NBL","SIN",1304.46241338626,0,0.888606378314601,0.0730967082076506,0.0382969134777483
+2011,"NBS","AFRMY",1007.19458217251,0.00631786884779677,0.508465565886657,0.422925929720447,0.0622906355450989
+2011,"NBS","AMCS",909.144153886733,0,0.367998198470403,0.292855238783156,0.339146562746441
+2011,"NBS","AMN",1191.16246661423,0.00175140501956247,0.246031544748117,0.0171544198963336,0.735062630335987
+2011,"NBS","ASIAUS",850.486998137522,0,0.1275802765743,0.675169487129654,0.197250236296046
+2011,"NBS","CEI",1510.19828714821,0.0265783802986438,0.0992704251101314,0.683706308679481,0.190444885911744
+2011,"NBS","CHN",1064.18012416966,0.00241012155494277,0.682256953411435,0.302010076448268,0.0133228485853545
+2011,"NBS","EUR",916.174649366482,0.0492043912001525,0.34135739820329,0.141561218830127,0.467876991766431
+2011,"NBS","SIN",888.171167446202,0,0.808119113541168,0.191880886458832,0
+2012,"NBI","AFRMY",1444.17076370054,0.0189785876610238,0.554143289263506,0.183522738833286,0.243355384242185
+2012,"NBI","AMCS",1145.36576413174,0.00268832200359789,0.559544366738983,0.142446977651729,0.29532033360569
+2012,"NBI","AMN",1720.16908219547,0.0159141747121521,0.677841464959253,0.0741658832141036,0.232078477114491
+2012,"NBI","ASIAUS",1085.95332163139,0.00265478449436225,0.237959011044416,0.220868274460398,0.538517930000824
+2012,"NBI","CEI",2006.06580045659,0.100906253687168,0.227465624809027,0.550683402218782,0.120944719285024
+2012,"NBI","CHN",1158.59266441247,0.0141035864977569,0.467190187861662,0.460395447099901,0.0583107785406796
+2012,"NBI","EUR",1182.16457725069,0.0968643146367463,0.30424724675862,0.0665441602095555,0.532344278395079
+2012,"NBI","SIN",1201.81633691092,0,0.341647722981543,0.145802256860062,0.512550020158395
+2012,"NBL","AFRMY",1608.76691303504,0.000106582842890438,0.787118825467425,0.212774591689684,0
+2012,"NBL","AMCS",1942.59130370118,0,0.632402926629166,0.367597073370834,0
+2012,"NBL","AMN",2247.80377072045,3.03863954678626e-05,0.9736440960383,0.0208958045695639,0.00542971299666854
+2012,"NBL","ASIAUS",949.291561304045,0,0.395181929394214,0.537425360098971,0.0673927105068143
+2012,"NBL","CEI",2327.82679505328,0,0.413095631019926,0.56467590472136,0.0222284642587132
+2012,"NBL","CHN",1293.31687328402,0,0.747038789170222,0.226671463593282,0.0262897472364964
+2012,"NBL","EUR",1552.95001340446,0.181635309992354,0.620577092099919,0.0768750158845336,0.120912582023194
+2012,"NBL","SIN",1262.1772234702,0,0.837584118852054,0.0666601175823454,0.0957557635656006
+2012,"NBS","AFRMY",1013.10920950682,0.000633311637145888,0.499427469358913,0.443650012538879,0.0562892064650616
+2012,"NBS","AMCS",917.64807277123,0,0.386175386765132,0.252805660269242,0.361018952965626
+2012,"NBS","AMN",1195.15696859212,0.00214482525496823,0.22493055282814,0.0176967797191902,0.755227842197701
+2012,"NBS","ASIAUS",753.026296589535,0,0.156786384849645,0.576596768499782,0.266616846650573
+2012,"NBS","CEI",1516.90709842238,0.0265719133759967,0.109642600306312,0.662408447449585,0.201377038868107
+2012,"NBS","CHN",1061.66838989265,0.0138058800383036,0.636560586041516,0.330225971057657,0.0194075628625229
+2012,"NBS","EUR",905.104659916776,0.0377161840217475,0.358327151010614,0.139953298748226,0.464003366219412
+2012,"NBS","SIN",840.89862762842,0,0.730587480131812,0.161983596208537,0.107428923659652
+2013,"NBI","AFRMY",1471.14511593237,0.0114910114769816,0.534102122708138,0.210037914839755,0.244368950975125
+2013,"NBI","AMCS",1155.06924658021,0.00241568258444271,0.543635960449871,0.136476587379677,0.317471769586009
+2013,"NBI","AMN",1719.64958082902,0.014770684837483,0.658952866924056,0.0664519605527871,0.259824487685674
+2013,"NBI","ASIAUS",1082.61894926893,0.00175837639481114,0.224484318647278,0.195343168934285,0.578414136023626
+2013,"NBI","CEI",1991.25170431686,0.0973485691261835,0.226934421221344,0.537183129176646,0.138533880475826
+2013,"NBI","CHN",1169.0751683354,0.0203698386923458,0.45928577064934,0.461191035794118,0.0591533548641968
+2013,"NBI","EUR",1194.77446120391,0.107305939312392,0.269210063850985,0.0765411714068999,0.546942825429723
+2013,"NBI","SIN",1189.66358932435,0,0.291268321817293,0.15977615942733,0.548955518755377
+2013,"NBL","AFRMY",1504.04783756136,0,0.798346292064315,0.201653707935685,0
+2013,"NBL","AMCS",1937.30127164138,0,0.663773719539538,0.336226280460462,0
+2013,"NBL","AMN",2286.21304965688,0.000211067046226865,0.955676846791949,0.0337984109773968,0.0103136751844274
+2013,"NBL","ASIAUS",946.036297823998,0,0.395066191903056,0.540170996680067,0.0647628114168767
+2013,"NBL","CEI",2188.51497455621,0,0.442662200849931,0.469005392742746,0.0883324064073231
+2013,"NBL","CHN",1285.8545792696,0,0.734483212310615,0.240666093799954,0.0248506938894303
+2013,"NBL","EUR",1532.39136571723,0.172350487079934,0.661374814189746,0.0501506017335714,0.116124096996749
+2013,"NBL","SIN",1328.54294126624,0,0.808486086695855,0.0986397949140966,0.092874118390048
+2013,"NBS","AFRMY",1068.69288183705,0.0021694248470834,0.483859805069158,0.462530097211811,0.0514406728719476
+2013,"NBS","AMCS",917.755280269507,0,0.37228855641885,0.278466499562154,0.349244944018996
+2013,"NBS","AMN",1189.93496263923,0.000811672645167246,0.237060773990007,0.0203814199708714,0.741746133393954
+2013,"NBS","ASIAUS",731.034760384142,0,0.135053042650456,0.550677194860484,0.31426976248906
+2013,"NBS","CEI",1422.79351241332,0.0308013295978799,0.111389044987001,0.653678245758535,0.204131379656584
+2013,"NBS","CHN",1081.8995362773,0.00914409729134034,0.623942014472763,0.346278836081969,0.0206350521539276
+2013,"NBS","EUR",898.461746765098,0.0395477275546877,0.366782374910371,0.13387853598876,0.459791361546181
+2013,"NBS","SIN",869.297618614158,0,0.679750084933829,0.247506035959847,0.0727438791063237
+2014,"NBI","AFRMY",1436.40944663271,0.0064222364775975,0.542811703744438,0.189062457146068,0.261703602631896
+2014,"NBI","AMCS",1170.64107322118,0.0027493707722935,0.530203693254464,0.135254397599347,0.331792538373895
+2014,"NBI","AMN",1727.68329190005,0.0178531848305123,0.637693960736605,0.0603331839078765,0.284119670525006
+2014,"NBI","ASIAUS",1069.76640284961,0.00140903943577794,0.219788280556817,0.199617160387617,0.579185519619788
+2014,"NBI","CEI",1973.25252699485,0.0911675719890466,0.292216237619569,0.482044785968709,0.134571404422675
+2014,"NBI","CHN",1178.42907769604,0.0203759433254584,0.454603685681043,0.45088118606297,0.074139184930529
+2014,"NBI","EUR",1220.87111343989,0.118961663081345,0.2616043762218,0.0718391226060263,0.547594838090828
+2014,"NBI","SIN",1186.49577387613,0,0.306002165399449,0.153511675513394,0.540486159087157
+2014,"NBL","AFRMY",1460.85102879045,0,0.799996324768136,0.200003675231864,0
+2014,"NBL","AMCS",1834.96900554596,0,0.71100178980825,0.28899821019175,0
+2014,"NBL","AMN",2268.84482457768,6.72936818232921e-05,0.934895447344904,0.0504745212860161,0.014562737687257
+2014,"NBL","ASIAUS",1000.51839657752,0,0.426399899813303,0.481541523775617,0.09205857641108
+2014,"NBL","CEI",2053.73479130168,0.000544108512580672,0.416405433989741,0.469643769666972,0.113406687830707
+2014,"NBL","CHN",1291.02642060912,0,0.716545024776569,0.255027082980876,0.028427892242555
+2014,"NBL","EUR",1606.64593197914,0.223161174507677,0.623883423471845,0.045626760908785,0.107328641111694
+2014,"NBL","SIN",1348.85019466439,0,0.765337403190089,0.115260651990989,0.119401944818922
+2014,"NBS","AFRMY",1064.45898081177,0.00291491760697259,0.485878275640673,0.457701466135592,0.0535053406167621
+2014,"NBS","AMCS",928.090241708309,0,0.411885967404546,0.272978645563733,0.315135387031722
+2014,"NBS","AMN",1205.70501230508,2.45836912217714e-05,0.260890569176629,0.0177039188823548,0.721380928249794
+2014,"NBS","ASIAUS",753.068372303079,0,0.096516532958733,0.566967056952962,0.336516410088305
+2014,"NBS","CEI",1431.03331915115,0.068921517641212,0.0570462073460887,0.677622808512896,0.196409466499804
+2014,"NBS","CHN",1105.47632702017,0.00768325406242105,0.570544252275754,0.401294897685053,0.0204775959767723
+2014,"NBS","EUR",914.25425692296,0.0393740037806408,0.356954682548805,0.141471803256619,0.462199510413936
+2014,"NBS","SIN",870.669726393812,0,0.733632170133234,0.222198134187043,0.0441696956797234
+2015,"NBI","AFRMY",1428.53050234763,0.0207766178438452,0.492908544562016,0.196885283444095,0.289429554150043
+2015,"NBI","AMCS",1203.29309803984,0.0028179607064242,0.520867760933472,0.140938655203608,0.335375623156496
+2015,"NBI","AMN",1738.01477644474,0.0238092584310295,0.579818612000771,0.0581504707429435,0.338221658825256
+2015,"NBI","ASIAUS",1052.60453445109,0.0032172314743161,0.219776081816767,0.185703310311,0.591303376397917
+2015,"NBI","CEI",1899.158400117,0.0893669146996939,0.289209803218771,0.462718555092102,0.158704726989434
+2015,"NBI","CHN",1218.17124952301,0.00914828266010726,0.441899152517771,0.461115758075308,0.0878368067468136
+2015,"NBI","EUR",1207.32369074129,0.118256348177102,0.250103229431232,0.059944525531618,0.571695896860048
+2015,"NBI","SIN",1176.24230287256,0,0.307018807724042,0.150213486045044,0.542767706230914
+2015,"NBL","AFRMY",1494.84589917535,0.0112118503311329,0.779272623184427,0.20951552648444,0
+2015,"NBL","AMCS",1562.20669493107,0,0.6986386766052,0.275581685391551,0.0257796380032494
+2015,"NBL","AMN",2268.33992274839,9.04679809181425e-06,0.911300397608658,0.06437449238171,0.0243160632115405
+2015,"NBL","ASIAUS",993.747398844302,0,0.467001449276421,0.399133992157238,0.133864558566342
+2015,"NBL","CEI",2048.75127359522,0,0.437091066089999,0.440660700787902,0.122248233122099
+2015,"NBL","CHN",1315.70922515485,0,0.690747469164544,0.275857121788021,0.0333954090474352
+2015,"NBL","EUR",1587.82150152401,0.221710041191847,0.624500798518349,0.0379757193450513,0.115813440944753
+2015,"NBL","SIN",1425.34077085995,0,0.73022044751683,0.173733140503727,0.0960464119794426
+2015,"NBS","AFRMY",1081.31463618046,0,0.502376778765073,0.441361634783227,0.0562615864516999
+2015,"NBS","AMCS",920.124003016477,0,0.408484604595118,0.273477618923622,0.31803777648126
+2015,"NBS","AMN",1207.20179372532,0.000510259136209996,0.260185914010707,0.0170901915972747,0.722213635255808
+2015,"NBS","ASIAUS",746.022483799258,0,0.0177077500445511,0.698664388961756,0.283627860993693
+2015,"NBS","CEI",1343.95715299032,0.0635179365450916,0.0472716426192377,0.693401532252457,0.195808888583214
+2015,"NBS","CHN",1108.44286330452,0.0776463771665144,0.489628594799918,0.411511072706446,0.0212139553271215
+2015,"NBS","EUR",939.017335617323,0.0313696377066949,0.393312783030407,0.12695669793992,0.448360881322978
+2015,"NBS","SIN",870.348448252073,0,0.702505145710871,0.172709655017247,0.124785199271883
+2016,"NBI","AFRMY",1475.56331850991,0.0319754311921306,0.501860657126291,0.183281816589932,0.282882095091646
+2016,"NBI","AMCS",1268.42203606365,0.0023583603231158,0.563185394601305,0.0808479769156193,0.35360826815996
+2016,"NBI","AMN",1711.79670365248,0.02577849127476,0.549989380291664,0.0492655076190842,0.374966620814492
+2016,"NBI","ASIAUS",1070.93354324963,0.00516606331186542,0.197972500385649,0.184155947112969,0.612705489189516
+2016,"NBI","CEI",1862.14713610059,0.112573334673903,0.33565303689593,0.379862621299724,0.171911007130443
+2016,"NBI","CHN",1245.23338795978,0.0174303355553323,0.410030769403998,0.489632917891661,0.0829059771490086
+2016,"NBI","EUR",1232.35805700697,0.123501446929874,0.227869683386626,0.0588770681164627,0.589751801567038
+2016,"NBI","SIN",1193.57262201123,0,0.289360253571763,0.140917456810684,0.569722289617552
+2016,"NBL","AFRMY",1547.9837934542,0.00152621623563369,0.848747591920331,0.149726191844036,0
+2016,"NBL","AMCS",1399.28855792509,0,0.798176059610811,0.11870362890116,0.0831203114880292
+2016,"NBL","AMN",2205.14765666623,0.000527650985546271,0.873246410525674,0.0677079740707337,0.058517964418046
+2016,"NBL","ASIAUS",1020.687695705,0,0.465360160672609,0.398125297827866,0.136514541499525
+2016,"NBL","CEI",2105.59709994094,0.000951451629755652,0.391067283495658,0.46983310993992,0.138148154934667
+2016,"NBL","CHN",1320.79179131628,0,0.639442580076711,0.347677415343407,0.012880004579882
+2016,"NBL","EUR",1554.64543886728,0.229774823410553,0.582330931609537,0.0419130744654885,0.145981170514422
+2016,"NBL","SIN",1460.91539363769,0,0.691968822226738,0.242406130669295,0.065625047103968
+2016,"NBS","AFRMY",1043.2327331395,0.000102620028115559,0.634683193935788,0.323485113563352,0.0417290724727441
+2016,"NBS","AMCS",1013.31700243345,0,0.501353450401791,0.189225496465874,0.309421053132335
+2016,"NBS","AMN",1225.63550236901,0.00141736160621114,0.289719174320785,0.0152546624653066,0.693608801607697
+2016,"NBS","ASIAUS",904.726098221287,0,0,0.798881873808401,0.201118126191599
+2016,"NBS","CEI",1364.60825764312,0.0683115316622174,0.00476814895964074,0.762177897951658,0.164742421426484
+2016,"NBS","CHN",1103.26586893804,0.00266194495735963,0.542546304930504,0.416096118847209,0.0386956312649273
+2016,"NBS","EUR",948.655111540345,0.0320549107778452,0.405305244353362,0.122519499361926,0.440120345506867
+2016,"NBS","SIN",844.823445399405,0,0.721656446373448,0.216189948982058,0.0621536046444943
+2017,"NBI","AFRMY",1496.17994808487,0.050625567469187,0.474891027462492,0.177206079246246,0.297277325822076
+2017,"NBI","AMCS",1340.77054209775,0.00419097560687048,0.548911692692836,0.0695792033443306,0.377318128355963
+2017,"NBI","AMN",1689.09690632216,0.0253778746560111,0.517462892568269,0.0468534041592378,0.410305828616482
+2017,"NBI","ASIAUS",1090.60485991004,0.00302765116934079,0.182402439784683,0.204964818235332,0.609605090810644
+2017,"NBI","CEI",1829.02413533484,0.122318230623105,0.335686084091387,0.365400712322871,0.176594972962637
+2017,"NBI","CHN",1241.32330932168,0.024165109264865,0.383933267779363,0.50615740525501,0.085744217700762
+2017,"NBI","EUR",1241.31006489195,0.123124647157609,0.21410854053978,0.0595471173091829,0.603219694993428
+2017,"NBI","SIN",1183.69574724329,0,0.242874340420892,0.166468473106022,0.590657186473086
+2017,"NBL","AFRMY",1555.18905266997,2.73547557210255e-06,0.898899901855501,0.101097362668927,0
+2017,"NBL","AMCS",1376.27783117168,0,0.706932971944753,0.0926412724368285,0.200425755618419
+2017,"NBL","AMN",2175.50832123322,0.00239165454393695,0.841933196283437,0.0707387560852616,0.0849363930873646
+2017,"NBL","ASIAUS",1023.43566831925,0.000800258513939384,0.447394398349715,0.414625939796609,0.137179403339737
+2017,"NBL","CEI",2180.61964841118,0.00560280639904956,0.410272233726201,0.435728583453766,0.148396376420984
+2017,"NBL","CHN",1336.04580667692,0,0.635925974829241,0.337062109484366,0.0270119156863931
+2017,"NBL","EUR",1581.27109738214,0.209323748985351,0.535854354427379,0.0590779284422201,0.19574396814505
+2017,"NBL","SIN",1385.85005953748,0,0.694349060550439,0.305650939449561,0
+2017,"NBS","AFRMY",1022.05829700248,0.000723919974063216,0.635702564857298,0.350461018515382,0.0131124966532562
+2017,"NBS","AMCS",944.049052843015,0.0115925636002999,0.489370351314295,0.208584407086816,0.29045267799859
+2017,"NBS","AMN",1209.28347755063,0.000260843711046378,0.301428731430571,0.0144515929155083,0.683858831942874
+2017,"NBS","ASIAUS",956.073385582094,0.00147769148061019,0,0.911135321358445,0.0873869871609443
+2017,"NBS","CEI",1339.60911044415,0.063829015346455,0,0.780514118854084,0.155656865799461
+2017,"NBS","CHN",1096.72095621103,0,0.495554779702135,0.461495254014362,0.0429499662835029
+2017,"NBS","EUR",962.176517730686,0.0334553700671753,0.404898911096585,0.113828314931266,0.447817403904974
+2017,"NBS","SIN",863.781882075268,0,0.778308570936917,0.221691429063083,0
+2018,"NBI","AFRMY",1549.53915639867,0.0663002134707632,0.449777947103253,0.172174303179534,0.31174753624645
+2018,"NBI","AMCS",1380.65995729124,0.00403791875092572,0.526953049921794,0.0644728642739742,0.404536167053306
+2018,"NBI","AMN",1681.37583151833,0.0243223958910481,0.492169303732863,0.0633986007298211,0.420109699646268
+2018,"NBI","ASIAUS",1102.63856456315,0.00113091958105788,0.161121175570221,0.216596903036671,0.621151001812049
+2018,"NBI","CEI",1838.97361970384,0.115750602261845,0.314124925171631,0.348725437026174,0.22139903554035
+2018,"NBI","CHN",1256.69089158885,0.0265087757771971,0.357904805904709,0.519799449653997,0.0957869686640962
+2018,"NBI","EUR",1267.88271268358,0.116072242775617,0.212225378892056,0.0546706111589407,0.617031767173387
+2018,"NBI","SIN",1192.02157391478,0,0.211194036778494,0.157406085488894,0.631399877732612
+2018,"NBL","AFRMY",1592.64169651201,0.0513972464483145,0.840367241579313,0.108235511972372,0
+2018,"NBL","AMCS",1340.46644169513,0,0.66054449860418,0.0732958526968107,0.266159648699009
+2018,"NBL","AMN",2183.23533315202,0.0017923290021823,0.823287104617353,0.0822383579702717,0.0926822084101931
+2018,"NBL","ASIAUS",1066.82729853101,0.000643403947630106,0.370208117432283,0.365332096984865,0.263816381635222
+2018,"NBL","CEI",2267.56279416812,0.0454603660327829,0.435043783737768,0.333858182214455,0.185637668014994
+2018,"NBL","CHN",1344.92957290438,0,0.611070288854834,0.344016836752947,0.0449128743922186
+2018,"NBL","EUR",1588.3174282926,0.18395427233798,0.527489434361314,0.0664491093108706,0.222107183989835
+2018,"NBL","SIN",1429.44110780293,0,0.727694458670705,0.272305541329295,0
+2018,"NBS","AFRMY",1020.71685747481,0.00692659851352168,0.648337238023146,0.339492629211395,0.00524353425193789
+2018,"NBS","AMCS",902.398858952222,0.00767826812173117,0.505309253487862,0.23799696604927,0.249015512341137
+2018,"NBS","AMN",1206.8536506272,0.000238267606518097,0.314470771627468,0.0207780560305486,0.664512904735466
+2018,"NBS","ASIAUS",911.614877157633,0,0.0279970581658189,0.88880103749632,0.0832019043378608
+2018,"NBS","CEI",1357.86587539802,0.0698812672610752,0,0.777867892979874,0.152250839759051
+2018,"NBS","CHN",1107.47879126715,0,0.492149708807731,0.486728197049722,0.0211220941425467
+2018,"NBS","EUR",970.85688005974,0.0343455036778129,0.41137643315386,0.120211212327105,0.434066850841222
+2018,"NBS","SIN",894.381759680565,0,0.855936877250193,0.144063122749808,0
+2019,"NBI","AFRMY",1492.2978111226,0.102649918856002,0.42680578751604,0.165936644969946,0.304607648658012
+2019,"NBI","AMCS",1354.78961410027,0.00337035610822788,0.470985961521779,0.0723116182708271,0.453332064099166
+2019,"NBI","AMN",1643.57578285843,0.0202378025679542,0.460651074647505,0.073868504257858,0.445242618526683
+2019,"NBI","ASIAUS",1134.24920064386,0.00108039379605935,0.135728909953501,0.224294044582557,0.638896651667882
+2019,"NBI","CEI",1813.05675723459,0.0896501184478546,0.309554188276996,0.323469457108335,0.277326236166814
+2019,"NBI","CHN",1271.78426912405,0.0248194088558872,0.37000220364988,0.509070546712698,0.0961078407815349
+2019,"NBI","EUR",1273.63382759207,0.107488921776733,0.217887890655653,0.0521920020091823,0.622431185558432
+2019,"NBI","SIN",1200.46166820718,0,0.0833726872437807,0.137415457334685,0.779211855421535
+2019,"NBL","AFRMY",1662.94590864627,0,0.842797719838686,0.124673635166073,0.0325286449952406
+2019,"NBL","AMCS",1235.12258886029,0,0.617596872702616,0.0477170231320532,0.334686104165331
+2019,"NBL","AMN",2148.05011529566,0.00761783833122888,0.802867121032521,0.098422138050584,0.0910929025856656
+2019,"NBL","ASIAUS",1144.60519451324,0,0.371336633430582,0.30078615964931,0.327877206920108
+2019,"NBL","CEI",2368.60926137144,0.0485422579137862,0.38017253853315,0.347589217922245,0.223695985630819
+2019,"NBL","CHN",1358.19649018895,0,0.645339552021407,0.310728603013136,0.0439318449654567
+2019,"NBL","EUR",1574.6849710538,0.150220914130494,0.551073518822584,0.0679955670446141,0.230710000002308
+2019,"NBL","SIN",1481.69541448869,0,0.694738182374634,0.220867504903163,0.0843943127222033
+2019,"NBS","AFRMY",1011.81873527508,0.0184179892903638,0.671794608859629,0.306147393397505,0.00364000845250183
+2019,"NBS","AMCS",918.820675598173,0,0.494093062827807,0.249364558198859,0.256542378973333
+2019,"NBS","AMN",1175.52459983989,0.00114786266440984,0.328186295175556,0.033747039008561,0.636918803151473
+2019,"NBS","ASIAUS",920.61931795843,0.000417544188915301,0.0591318440835829,0.860276886661068,0.0801737250664342
+2019,"NBS","CEI",1389.23206478903,0.0789510027434591,0,0.773127726131804,0.147921271124737
+2019,"NBS","CHN",1114.70988332997,0,0.455976409034769,0.507095166014595,0.0369284249506358
+2019,"NBS","EUR",924.972539854576,0.0298135908183802,0.427911639420309,0.0787710251981705,0.46350374456314
+2019,"NBS","SIN",846.981601236901,0,0.868729299147553,0.131270700852447,0
+2020,"NBI","AFRMY",1426.25559102747,0.106848432206768,0.373383866727221,0.179912610347422,0.339855090718589
+2020,"NBI","AMCS",1306.44021925163,8.79269989742728e-05,0.365998869543555,0.0756483357196987,0.558264867737772
+2020,"NBI","AMN",1607.55973458849,0.0208137079668084,0.404852448191474,0.0565542123531072,0.51777963148861
+2020,"NBI","ASIAUS",985.261328996938,0.00163127066622938,0.136321884296025,0.299521925183548,0.562524919854198
+2020,"NBI","CEI",1723.1238461104,0.0127266489771093,0.223472267070623,0.300583795139888,0.46321728881238
+2020,"NBI","CHN",1206.22329925023,0.0172890851938207,0.357717276826464,0.522238373766762,0.102755264212954
+2020,"NBI","EUR",1222.70165889077,0.108458721201359,0.222676388841383,0.0504372290592626,0.618427660897995
+2020,"NBI","SIN",1184.95872053627,0,0.0440019446739351,0.11774326558819,0.838254789737875
+2020,"NBL","AFRMY",1482.67185044828,0.000351746209561387,0.717985880116287,0.135904425201487,0.145757948472665
+2020,"NBL","AMCS",1276.0795553978,0,0.443361917669241,0.0259725624881629,0.530665519842596
+2020,"NBL","AMN",2049.02352240078,0.0118712033288142,0.772425051283519,0.120716504974669,0.0949872404129974
+2020,"NBL","ASIAUS",899.538384968181,0,0.333638225914751,0.361960367141017,0.304401406944233
+2020,"NBL","CEI",2148.90533738309,0.043822817946538,0.230190185738548,0.39626685985108,0.329720136463834
+2020,"NBL","CHN",1285.39430942117,0,0.660866332697191,0.325647988412346,0.0134856788904631
+2020,"NBL","EUR",1546.74022682415,0.0507529300296071,0.575646353351569,0.0477569957210068,0.325843720897817
+2020,"NBL","SIN",1355.02510680111,0,0.577504652539615,0.150945452269486,0.2715498951909
+2020,"NBS","AFRMY",935.255247110228,0.0199587016443027,0.666753484356739,0.313287813998959,0
+2020,"NBS","AMCS",902.162552916117,0,0.43782454377697,0.190070588729677,0.372104867493353
+2020,"NBS","AMN",1156.20881319321,7.74534625298871e-06,0.350232699958734,0.0256591015097475,0.624100453185266
+2020,"NBS","ASIAUS",825.335039912212,0,0.0562683451124768,0.908293639144726,0.0354380157427969
+2020,"NBS","CEI",1465.78858435796,0.106767581999736,0,0.807299576017807,0.0859328419824571
+2020,"NBS","CHN",1053.99157417391,0,0.406363801780305,0.517106893193666,0.0765293050260291
+2020,"NBS","EUR",881.94342382916,0.0369041996788106,0.497107719421705,0.0788573689914357,0.387130711908049
+2020,"NBS","SIN",842.928772674598,0,0.865726665502825,0.134273334497175,0
+2021,"NBI","AFRMY",1414.88415520516,0.0929941722087423,0.354796579859671,0.18322125464213,0.368987993289457
+2021,"NBI","AMCS",1224.77547703967,0,0.388189694550877,0.0706241741032749,0.541186131345848
+2021,"NBI","AMN",1633.84097570581,0.00803674285865951,0.399727129589615,0.0547820246268294,0.537454102924896
+2021,"NBI","ASIAUS",865.593828023355,0.00359077348539316,0.130402188765361,0.34989126700319,0.516115770746056
+2021,"NBI","CEI",1715.86698426312,0.0179473941125914,0.211494434523127,0.289180322252333,0.481377849111949
+2021,"NBI","CHN",1196.64231448929,0.0206478966337006,0.349782838674386,0.548240975791555,0.0813282889003586
+2021,"NBI","EUR",1266.71603824698,0.111288729111577,0.231708502222202,0.0535219844845168,0.603480784181704
+2021,"NBI","SIN",1168.04223850818,0,0.0436943252376604,0.129403678975315,0.826901995787025
+2021,"NBL","AFRMY",1451.47763124676,0.00138482532298442,0.559998858896625,0.230461545327438,0.208154770452952
+2021,"NBL","AMCS",1426.70968494685,0,0.543301799558644,0.039099762091056,0.4175984383503
+2021,"NBL","AMN",1999.16939747984,0.00485818950836252,0.773041430291726,0.112015224939166,0.110085155260746
+2021,"NBL","ASIAUS",790.951652462864,0.00108618084020231,0.357002562371498,0.369735580100665,0.272175676687635
+2021,"NBL","CEI",1972.37515036087,0.0804303432954618,0.233857944276774,0.334786089398675,0.350925623029089
+2021,"NBL","CHN",1303.74363457761,0,0.70398999555753,0.264253203455216,0.0317568009872544
+2021,"NBL","EUR",1561.97812541815,0.0611927746145145,0.545851335831636,0.0804972976250072,0.312458591928842
+2021,"NBL","SIN",1148.27920897887,0,0.441275678262291,0.113636648112906,0.445087673624803
+2021,"NBS","AFRMY",856.192787702671,0.00287971393527096,0.550560978765089,0.44655930729964,0
+2021,"NBS","AMCS",846.498287878333,0,0.458158478997787,0.146770944356393,0.39507057664582
+2021,"NBS","AMN",1217.73804339635,0.000479895470863492,0.374176598876324,0.0178203623503828,0.607523143302429
+2021,"NBS","ASIAUS",785.74421816884,0.00558084459838678,0.0558453523727677,0.878736357466402,0.059837445562444
+2021,"NBS","CEI",1543.1603219169,0.102608666329066,0,0.857339513894178,0.040051819776756
+2021,"NBS","CHN",1030.93692671912,0,0.512164125718881,0.417343964011558,0.0704919102695618
+2021,"NBS","EUR",936.127412326756,0.0430143716604469,0.489084103918227,0.0888290264826769,0.379072497938649
+2021,"NBS","SIN",835.916471702758,0,0.861926075951236,0.138073924048764,0
+2022,"NBI","AFRMY",1508.02793033993,0.0928239959096221,0.351348406690954,0.191833274953088,0.363994322446336
+2022,"NBI","AMCS",1253.60867656486,0.00155004878751767,0.414901184784682,0.079077187632123,0.504471578795677
+2022,"NBI","AMN",1603.84210069102,0.0233107034422129,0.399195566635614,0.0479893257361896,0.529504404185984
+2022,"NBI","ASIAUS",940.912955028639,0.0205676350844789,0.107599193269436,0.313939305146886,0.5578938664992
+2022,"NBI","CEI",1745.27844502451,0.00833915599876527,0.18921846489797,0.369509956297374,0.43293242280589
+2022,"NBI","CHN",1254.89850091034,0.0177947939078773,0.347283014203228,0.534906235380386,0.100015956508509
+2022,"NBI","EUR",1323.69990096757,0.0954795385367953,0.230089479151201,0.0519695510635642,0.622461431248439
+2022,"NBI","SIN",1243.99970231986,0,0.0592884712242457,0.137917967629151,0.802793561146603
+2022,"NBL","AFRMY",1676.60937263212,0.00281294148250758,0.534873022503775,0.143288502499541,0.319025533514176
+2022,"NBL","AMCS",1371.35837291087,8.20082972860506e-05,0.562808807640177,0.0380541338860963,0.399055050176441
+2022,"NBL","AMN",2054.24043496376,0.0183217001977137,0.760127960018196,0.125959978586627,0.0955903611974634
+2022,"NBL","ASIAUS",915.295801823025,0.00305899241747255,0.335999700744929,0.294693633557543,0.366247673280056
+2022,"NBL","CEI",2064.37589694466,0.0935933793784874,0.163971142624411,0.468512765498142,0.27392271249896
+2022,"NBL","CHN",1342.72615190688,0,0.721492954093771,0.242822087815385,0.0356849580908439
+2022,"NBL","EUR",1596.1270790658,0.0484788557290657,0.504223696059201,0.0606250058690828,0.38667244234265
+2022,"NBL","SIN",1214.63579642949,0,0.26197395277676,0.0981350053960581,0.639891041827182
+2022,"NBS","AFRMY",911.09338630692,0.00351769806258936,0.522374745960725,0.474107555976685,0
+2022,"NBS","AMCS",884.066743879667,0,0.575797186859166,0.145484987711224,0.27871782542961
+2022,"NBS","AMN",1158.47522743255,0.00374269998456825,0.396360775933847,0.0277995351662053,0.572096988915379
+2022,"NBS","ASIAUS",772.956060589141,0.00303386975872505,0.0530916415636241,0.828525538962808,0.115348949714842
+2022,"NBS","CEI",1589.78033325245,0.111400419373622,0,0.878810147303776,0.00978943332260171
+2022,"NBS","CHN",1085.29108387524,0,0.508404168500958,0.427733047993193,0.0638627835058491
+2022,"NBS","EUR",935.501417022643,0.0497429984106455,0.363821183684826,0.0810580895792733,0.505377728325256
+2022,"NBS","SIN",951.068123227409,0,0.920785060368375,0.0792149396316247,0
+2023,"NBI","AFRMY",1556.80725540075,0.108705729988229,0.357505917314596,0.189058468969886,0.344729883727289
+2023,"NBI","AMCS",1241.80819937441,0.0289649721176391,0.41397210082774,0.108470008003653,0.448592919050968
+2023,"NBI","AMN",1564.0567787468,0.0361240344242864,0.382880330125865,0.0376911413380658,0.543304494111783
+2023,"NBI","ASIAUS",1060.77102245777,0.0434597096601014,0.0996079028519961,0.263029878790639,0.593902508697264
+2023,"NBI","CEI",1826.74187551772,0.0161406537098493,0.181865607729458,0.451368477975653,0.35062526058504
+2023,"NBI","CHN",1283.76020047764,0.0216716274453564,0.351912323428936,0.531434528290253,0.0949815208354548
+2023,"NBI","EUR",1317.06560783486,0.107020107150277,0.228792064406469,0.0584522649085934,0.60573556353466
+2023,"NBI","SIN",1270.1874747079,0.00126325286810533,0.111853127830007,0.164994711115334,0.721888908186554
+2023,"NBL","AFRMY",1800.06458633877,0.0133801106637488,0.471638834214414,0.123224660013843,0.391756395107995
+2023,"NBL","AMCS",1471.55344394088,0.000357247706404697,0.514015409920159,0.0522471049169055,0.43338023745653
+2023,"NBL","AMN",2039.1261458019,0.0196181402759156,0.742988516683088,0.135165871294862,0.102227471746134
+2023,"NBL","ASIAUS",1166.41083664536,0.000821334636392441,0.337293163392121,0.258604343917637,0.403281158053849
+2023,"NBL","CEI",2075.76750196921,0.0793214969402916,0.155488785662149,0.497588719221434,0.267600998176125
+2023,"NBL","CHN",1406.90240627733,0,0.664221019460596,0.269499070261477,0.066279910277927
+2023,"NBL","EUR",1602.96286865254,0.0533530583431029,0.496518833895994,0.0684374957893583,0.381690611971545
+2023,"NBL","SIN",1207.05467159359,0.0144262416321987,0.100990835158525,0.0681571485731707,0.816425774636106
+2023,"NBS","AFRMY",948.355103599086,0.037007108754676,0.529137242613583,0.430945058144917,0.00291059048682419
+2023,"NBS","AMCS",816.632932817499,0.0403905851219203,0.543118516223362,0.128171678582223,0.288319220072495
+2023,"NBS","AMN",1161.48843882446,0.00544690110891447,0.397181519896337,0.0498032125672708,0.547568366427478
+2023,"NBS","ASIAUS",811.927507390515,0.00682706917018166,0.10350473181388,0.707316448782261,0.182351750233678
+2023,"NBS","CEI",1588.80512656602,0.0813996913130536,0,0.897450495806196,0.0211498128807507
+2023,"NBS","CHN",1146.74694167458,0,0.509160638468378,0.441672931770871,0.0491664297607512
+2023,"NBS","EUR",920.308789611741,0.0388582555858133,0.365848709273328,0.103845958454748,0.49144707668611
+2023,"NBS","SIN",878.739435661258,0,0.775025109785864,0.224974890214136,0
+2024,"NBI","AFRMY",1579.48948598743,0.106926908280134,0.360693015920477,0.189859687386662,0.342520388412728
+2024,"NBI","AMCS",1250.24403101588,0.0414159705614002,0.42689506003781,0.115876538651537,0.415812430749253
+2024,"NBI","AMN",1572.67501442444,0.0290876582832362,0.409449876448866,0.0362810938851846,0.525181371382713
+2024,"NBI","ASIAUS",1102.07739212829,0.0416770563805044,0.115549021444569,0.228050703639522,0.614723218535404
+2024,"NBI","CEI",1866.83639957087,0.0189098698362885,0.222097297249686,0.458414916894742,0.300577916019284
+2024,"NBI","CHN",1312.64618285207,0.0249347736528903,0.351847357160196,0.518839729058085,0.104378140128828
+2024,"NBI","EUR",1336.27890873247,0.117060447025523,0.221219359164198,0.0519930911985262,0.609727102611752
+2024,"NBI","SIN",1299.65489058005,0.00403312317372397,0.131350613355621,0.246597986209749,0.618018277260906
+2024,"NBL","AFRMY",1813.16505531261,0.0272968123923097,0.504343445099653,0.119369542405468,0.348990200102569
+2024,"NBL","AMCS",1565.72427872587,4.64099990715566e-05,0.556344879125429,0.0858793866552136,0.357729324220286
+2024,"NBL","AMN",2016.70289863956,0.0190064682639889,0.729746049435679,0.133516084090153,0.117731398210178
+2024,"NBL","ASIAUS",1247.63101805903,0,0.319612032106869,0.237939997376194,0.442447970516937
+2024,"NBL","CEI",2129.65465842982,0.0947184787619097,0.199795510901423,0.391833862691685,0.313652147644982
+2024,"NBL","CHN",1439.87320871325,0,0.618327064319791,0.282286201130497,0.0993867345497127
+2024,"NBL","EUR",1582.37900032054,0.0495865933513587,0.524625418338014,0.0561882336431036,0.369599754667523
+2024,"NBL","SIN",1227.30839530482,0.0160878020325343,0.0980672061606923,0.0544868664544921,0.831358125352281
+2024,"NBS","AFRMY",974.930683434649,0.0109738988294409,0.472691828022828,0.516334273147731,0
+2024,"NBS","AMCS",832.631287060762,0.0199417808883177,0.523428155173699,0.116230273314328,0.340399790623655
+2024,"NBS","AMN",1186.58632652511,0.00575161404328503,0.383000903445285,0.0665835446228188,0.544663937888611
+2024,"NBS","ASIAUS",812.959679368,0.00582811965677951,0.0902622565260704,0.756055039507645,0.147854584309506
+2024,"NBS","CEI",1639.1455889578,0.123707299379575,0,0.861039993258657,0.0152527073617684
+2024,"NBS","CHN",1167.96541144834,0,0.499883693867715,0.446786177395235,0.0533301287370499
+2024,"NBS","EUR",931.780060361597,0.0332866124235474,0.389731268275063,0.127345695393135,0.449636423908255
+2024,"NBS","SIN",1120.54617865243,0.0205221386317005,0.540682750051481,0.438795111316819,0

M2/Generative AI/TP1/TP1 Benchmark.ipynb

@@ -0,0 +1,851 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "172a7a9f",
+   "metadata": {},
+   "source": [
+    "# TP2 - Benchmark automatique\n",
+    "\n",
+    "Dans ce TP nous allons définir une fonction pour mesurer les performances d'un modèle de langage via l'exécution de plusieurs benchmarks. Nous avons vu en cours trois manières de mesurer la performance d'un modèle de langage qu'on peut résumer à:\n",
+    "1. **Évaluation automatique**: via un ensemble de questions dont on connait la réponse\n",
+    "2. **Évaluation humaine**: qualification humaine de la réponse d'un modèle à une question\n",
+    "3. **Évaluation par modèle de langage**: notation ou comparaison de réponse d'un ou plusieurs modèles par un autre modèle\n",
+    "\n",
+    "Nous nous intéressons ici au premier point, en particulier avec les benchmarks [GSM8K](https://huggingface.co/datasets/openai/gsm8k) et [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag).\n",
+    "Dans l'ensemble du notebook nous utiliserons la librairie LangChain.\n",
+    "\n",
+    "Il est à garder en tête que ce notebook n'a qu'une portée pédagogique et n'est pas forcément à jour puisque le domaine évolue rapidement, ni que les pratiques sont celles validées par l'industrie.\n",
+    "\n",
+    "## Uniformisation des benchmarks\n",
+    "\n",
+    "Pour chaque benchmark que l'on considère, nous avons besoin de plusieurs informations :\n",
+    "* **Dataset** : une fonction pour charger les questions du benchmark\n",
+    "* **Référence** : une fonction capable d'identifier la réponse attentue\n",
+    "* **Prompt** : un prompt qui permet de demander correctement au modèle de répondre à la question\n",
+    "* **Chaîne** : une fonction qui renvoie la chaîne de traitement de LangChain\n",
+    "* **Score** : une fonction qui score la performance d'un modèle sur une question\n",
+    "\n",
+    "Nous allons commencer par créer une classe qui regroupe ces desiderata :\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd75374d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.prompts import PromptTemplate\n",
+    "from langchain_core.runnables import Runnable\n",
+    "\n",
+    "\n",
+    "class Benchmark:\n",
+    "    \"\"\"Base class for benchmarks.\"\"\"\n",
+    "\n",
+    "    name: str\n",
+    "\n",
+    "    def __init__(self, prompt: PromptTemplate) -> None:\n",
+    "        \"\"\"Initialize the benchmark with a prompt template.\"\"\"\n",
+    "        self.prompt = prompt\n",
+    "\n",
+    "    def load_data(self) -> list:\n",
+    "        \"\"\"Load and return the benchmark data samples.\"\"\"\n",
+    "        raise NotImplementedError\n",
+    "\n",
+    "    def build_chain(self, model) -> Runnable:\n",
+    "        \"\"\"Build and return the evaluation chain using the provided model.\"\"\"\n",
+    "        raise NotImplementedError\n",
+    "\n",
+    "    def get_reference(self, sample) -> str:\n",
+    "        \"\"\"Extract and return the reference answer from a data sample.\"\"\"\n",
+    "        raise NotImplementedError\n",
+    "\n",
+    "    def score(self, prediction, reference) -> float:\n",
+    "        \"\"\"Score the prediction against the reference answer.\"\"\"\n",
+    "        raise NotImplementedError"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e2ab41df",
+   "metadata": {},
+   "source": [
+    "Pour rendre cette classe plus concrète, commençons par travailler avec le benchmark [GSM8K](https://huggingface.co/datasets/openai/gsm8k).\n",
+    "\n",
+    "### Benchmark GSM8K\n",
+    "\n",
+    "On commence par charger le dataset et observer une question.\n",
+    "\n",
+    "**Consigne** : Résoudre la question *à la main* et vérifier votre réponse. On recommande d'explorer plusieurs questions."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "93979ba0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of questions: 1319\n",
+      "Example of question:\n",
+      " Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?\n",
+      "And its answer:\n",
+      " Janet sells 16 - 3 - 4 = <<16-3-4=9>>9 duck eggs a day.\n",
+      "She makes 9 * 2 = $<<9*2=18>>18 every day at the farmer’s market.\n",
+      "#### 18\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "np.random.seed(42)\n",
+    "\n",
+    "dataset = load_dataset(\"gsm8k\", \"main\")\n",
+    "dataset = dataset[\"test\"]\n",
+    "\n",
+    "print(f\"Number of questions: {len(dataset)}\")\n",
+    "index = 0\n",
+    "print(\"Example of question:\\n\", dataset[index][\"question\"])\n",
+    "print(\"And its answer:\\n\", dataset[index][\"answer\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "82d797f0",
+   "metadata": {},
+   "source": [
+    "Après avoir inspecté plusieurs éléments du dataset, on remarque que la réponse finale est placée après la chaîne de caractères \"####\".\n",
+    "\n",
+    "**Consigne**: Construire une fonction `get_reference` qui prend en argument un élément de GMS8K (dictionnaire avec question et réponse) et renvoie la réponse attendue (string). On pourra utiliser la fonction [`search`](https://docs.python.org/3/library/re.html#re.search) de la librairie [`re`](https://docs.python.org/3/library/re.html#).\n",
+    "Puis tester cette fonction sur l'exemple précédent."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b336056a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Reference: 18\n"
+     ]
+    }
+   ],
+   "source": [
+    "from re import search\n",
+    "\n",
+    "\n",
+    "def get_reference(sample: dict) -> str:\n",
+    "    \"\"\"Extract the reference answer from a data sample.\"\"\"\n",
+    "    match = search(r\"#### (\\d+)\", sample[\"answer\"])\n",
+    "    return match.group(1) if match else None\n",
+    "\n",
+    "\n",
+    "index = 0\n",
+    "reference = get_reference(sample=dataset[index])\n",
+    "print(f\"Reference: {reference}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4c137e6a",
+   "metadata": {},
+   "source": [
+    "Il nous reste maintenant à définir un prompt tel que l'on puisse appeler un modèle et tester notre mécanique."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0b899872",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.prompts import PromptTemplate\n",
+    "\n",
+    "prompt = PromptTemplate(\n",
+    "    input_variables=[\"question\"],\n",
+    "    template=(\n",
+    "        \"\"\"You are a careful mathematician. Solve the problem step by step, then display your answer in the end.\n",
+    "        Question: {question}\n",
+    "        Answer:\"\"\"\n",
+    "    ),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "36433b53",
+   "metadata": {},
+   "source": [
+    "En intégrant l'appel à un modèle via Ollama sur notre ordinateur, on peut définir avec LangChain :"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f0676b6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Model answer : Here's how we can solve this problem step by step:\n",
+      "\n",
+      "1. **Calculate the total number of eggs laid:** Janet's ducks lay 16 eggs per day.\n",
+      "\n",
+      "2. **Calculate the number of eggs eaten:** She eats 3 eggs per day.\n",
+      "\n",
+      "3. **Calculate the number of eggs remaining after breakfast:** 16 eggs (laid) - 3 eggs (eaten) = 13 eggs\n",
+      "\n",
+      "4. **Calculate the number of eggs used for baking:** She uses 4 eggs for baking.\n",
+      "\n",
+      "5. **Calculate the number of eggs remaining after baking:** 13 eggs - 4 eggs (baking) = 9 eggs\n",
+      "\n",
+      "6. **Calculate the earnings from selling the remaining eggs:** She sells 9 eggs at $2 per egg.  So she makes 9 * $2 = $18.\n",
+      "\n",
+      "**Answer:** $18\n",
+      "The answer was : 18\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.runnables import RunnablePassthrough\n",
+    "from langchain_ollama import OllamaLLM\n",
+    "\n",
+    "model = OllamaLLM(model=\"gemma3:4b\")\n",
+    "\n",
+    "chain = {\"question\": RunnablePassthrough()} | prompt | model | StrOutputParser()\n",
+    "\n",
+    "index = 0\n",
+    "\n",
+    "question = dataset[index][\"question\"]\n",
+    "answer = get_reference(dataset[index])\n",
+    "response = chain.invoke(question)\n",
+    "print(f\"Model answer : {response}\")\n",
+    "print(f\"The answer was : {answer}\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "97dd7db7",
+   "metadata": {},
+   "source": [
+    "Il nous faut extraire la dernière valeur numérique pour obtenir automatiquement la réponse du modèle.\n",
+    "\n",
+    "**Consigne** : Définir une fonction `score` qui prend en paramètre la réponse du modèle et la réponse attendue puis renvoie si les deux réponses sont identiques (1 / 0). On pourra utiliser la fonction [`findall`](https://docs.python.org/3/library/re.html#re.findall) de la librairie `re`.\n",
+    "Puis l'appliquer sur l'exemple précédent."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ad43cf84",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The model scored 1.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from re import findall\n",
+    "\n",
+    "\n",
+    "def score(prediction, reference):\n",
+    "    if reference is None:\n",
+    "        return 0.0\n",
+    "\n",
+    "    numbers = findall(r\"\\d+\", prediction)\n",
+    "    return 1.0 if numbers and numbers[-1] == reference else 0.0\n",
+    "\n",
+    "\n",
+    "value = score(response, answer)\n",
+    "print(f\"The model scored {value}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a2ec5088",
+   "metadata": {},
+   "source": [
+    "Nous avons l'ensemble des éléments nécessaire pour définir la classe `GSM8KBenchmark` depuis la classe `Benchmark` que nous avons défini précédemment.\n",
+    "\n",
+    "**Consigne** : Définir cette classe comme sous-classe de `Benchmark`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "d83f4394",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class GSM8KBenchmark(Benchmark):\n",
+    "    name = \"GSM8K\"\n",
+    "\n",
+    "    def load_data(self):\n",
+    "        return load_dataset(\"gsm8k\", \"main\", split=\"test\")\n",
+    "\n",
+    "    def build_chain(self, model):\n",
+    "        return (\n",
+    "            {\"question\": RunnablePassthrough()}\n",
+    "            | self.prompt\n",
+    "            | model\n",
+    "            | StrOutputParser()\n",
+    "        )\n",
+    "\n",
+    "    def get_reference(self, sample):\n",
+    "        match = search(r\"#### (\\d+)\", sample[\"answer\"])\n",
+    "        return match.group(1) if match else None\n",
+    "\n",
+    "    def score(self, prediction, reference):\n",
+    "        if reference is None:\n",
+    "            return 0.0\n",
+    "        numbers = findall(r\"\\d+\", prediction)\n",
+    "        return 1.0 if numbers and numbers[-1] == reference else 0.0"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dfc3cb78",
+   "metadata": {},
+   "source": [
+    "Il est maintenant temps de définir une fonction qui *fait* le benchmark.\n",
+    "\n",
+    "**Consigne** : Définir une fonction `run_benchmark` qui prend en paramètre :\n",
+    "* `model_name` : le nom du modèle Ollama que l'on veut tester\n",
+    "* `benchmark` : la classe benchmark que l'on souhaite tester\n",
+    "* `max_samples` : le nombre maximum de questions que l'on souhaite utiliser\n",
+    "\n",
+    "Puisque l'object avec lequel nous travaillons est un dataset HuggingFace, pour sélectionner $n$ lignes, on utilisera \n",
+    "```python\n",
+    "dataset = dataset.select(range(max_samples))\n",
+    "```\n",
+    "De cette manière on préserve la structure."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2d7125af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm\n",
+    "import numpy as np\n",
+    "\n",
+    "\n",
+    "def run_benchmark(\n",
+    "    model_name: str, benchmark: Benchmark, max_samples: int | None = None\n",
+    ") -> dict:\n",
+    "    model = OllamaLLM(model=model_name)\n",
+    "\n",
+    "    data = benchmark.load_data()\n",
+    "    if max_samples:\n",
+    "        data = data.select(range(max_samples))\n",
+    "    chain = benchmark.build_chain(model)\n",
+    "\n",
+    "    scores = []\n",
+    "\n",
+    "    for sample in tqdm(data, desc=f\"Running {benchmark.name}\"):\n",
+    "        prediction = chain.invoke(sample)\n",
+    "        reference = benchmark.get_reference(sample)\n",
+    "        scores.append(benchmark.score(prediction, reference))\n",
+    "\n",
+    "    results = {\n",
+    "        \"benchmark\": benchmark.name,\n",
+    "        \"model\": model_name,\n",
+    "        \"num_samples\": len(scores),\n",
+    "        \"accuracy\": np.mean(scores),\n",
+    "    }\n",
+    "    return results\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "81de8940",
+   "metadata": {},
+   "source": [
+    "**Consigne** : Utiliser la fonction `run_benchmark` en définissant un prompt pour GSM8K."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f6bbeb53",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Running GSM8K: 100%|██████████| 5/5 [00:50<00:00, 10.18s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'benchmark': 'GSM8K', 'model': 'gemma3:4b', 'num_samples': 5, 'accuracy': 0.8}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "prompt_GMS8K = PromptTemplate(\n",
+    "    input_variables=[\"question\"],\n",
+    "    template=(\n",
+    "        \"\"\"You are a careful mathematician. Solve the problem step by step, then display your answer in the end.\n",
+    "        Question: {question}\n",
+    "        Answer:\"\"\"\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "benchmark_GSM8K = GSM8KBenchmark(prompt=prompt_GMS8K)\n",
+    "results = run_benchmark(\n",
+    "    model_name=\"gemma3:4b\", benchmark=benchmark_GSM8K, max_samples=5\n",
+    ")\n",
+    "print(results)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c943124",
+   "metadata": {},
+   "source": [
+    "### HellaSwag\n",
+    "\n",
+    "Maintenant que nous avons réussi à le faire pour le dataset GMS8K, attaquons-nous à [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag).\n",
+    "\n",
+    "**Consigne** : En suivant la même approche que précédemment, implémenter une sous classe `HellaSwagBenchmark` à partir de la classe `Benchmark`. Puis utiliser la fonction `run_benchmark` pour valider votre travail."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "32886901",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class HellaSwagBenchmark(Benchmark):\n",
+    "    name = \"HellaSwag\"\n",
+    "\n",
+    "    def load_data(self):\n",
+    "        return load_dataset(\"hellaswag\", split=\"validation\")\n",
+    "\n",
+    "    def build_chain(self, model):\n",
+    "        return (\n",
+    "            {\n",
+    "                \"context\": lambda x: x[\"ctx\"],\n",
+    "                \"choices\": lambda x: \"\\n\".join(\n",
+    "                    f\"{index}: {choice}\" for index, choice in enumerate(x[\"endings\"])\n",
+    "                ),\n",
+    "            }\n",
+    "            | self.prompt\n",
+    "            | model\n",
+    "            | StrOutputParser()\n",
+    "        )\n",
+    "\n",
+    "    def get_reference(self, sample):\n",
+    "        return str(sample[\"label\"])\n",
+    "\n",
+    "    def score(self, prediction, reference):\n",
+    "        match = search(r\"\\d\", prediction)\n",
+    "        return 1.0 if match and match.group(0) == reference else 0.0\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96a3031a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Running HellaSwag: 100%|██████████| 5/5 [00:02<00:00,  2.08it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'benchmark': 'HellaSwag', 'model': 'gemma3:4b', 'num_samples': 5, 'accuracy': 1.0}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "prompt_HellaSwag = PromptTemplate(\n",
+    "    input_variables=[\"context\", \"choices\"],\n",
+    "    template=(\n",
+    "        \"\"\"You will be given a context and then different choices. You need to find the most likely continuation to the context. Answer with the number of the most likely choice only.\n",
+    "        Context: {context}\n",
+    "        Choices: {choices}\n",
+    "        Answer:\"\"\"\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "benchmark_HellaSwag = HellaSwagBenchmark(prompt=prompt_HellaSwag)\n",
+    "\n",
+    "results = run_benchmark(\n",
+    "    model_name=\"gemma3:4b\", benchmark=benchmark_HellaSwag, max_samples=5\n",
+    ")\n",
+    "print(results)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c542783c",
+   "metadata": {},
+   "source": [
+    "## Réponses structurées\n",
+    "\n",
+    "Sur quelques exemples tout semble fonctionner ! Mais il y a au moins une fragilité dans notre travail : la récupération de la réponse est peu fiable et largement dépendante des prompts.\n",
+    "\n",
+    "\n",
+    "Par exemple pour GMS8K, on aimerait avoir une réponse sous la forme d'un JSON :\n",
+    "```json\n",
+    "{\n",
+    "  \"reasoning\": \"étapes de raisonnement\",\n",
+    "  \"final_answer\": 18\n",
+    "}\n",
+    "```\n",
+    "\n",
+    "De cette manière ce serait particulièrement simple d'extraire la réponse, sans pour autant ne pas avoir de *réflexion* du modèle. En revanche pour HellaSwag, un JSON extrêment simple suffit :\n",
+    "```json\n",
+    "{\n",
+    "    \"choice\": 2\n",
+    "}\n",
+    "```\n",
+    "\n",
+    "Pour forcer le modèle à suivre ces formats, nous allons utiliser l'option [Pydantic](https://docs.langchain.com/oss/python/langchain/structured-output). Elle s'utilise comme suit, pour GSM8K :"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "988dbca3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel, Field\n",
+    "\n",
+    "\n",
+    "class GSM8KOutput(BaseModel):\n",
+    "    reasoning: str = Field(description=\"Step-by-step reasoning\")\n",
+    "    final_answer: float = Field(description=\"Final numeric answer\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d855adfe",
+   "metadata": {},
+   "source": [
+    "Concernant l'intégration dans le prompt :"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f25afddc",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The output should be formatted as a JSON instance that conforms to the JSON schema below.\n",
+      "\n",
+      "As an example, for the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, \"required\": [\"foo\"]}\n",
+      "the object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} is not well-formatted.\n",
+      "\n",
+      "Here is the output schema:\n",
+      "```\n",
+      "{\"properties\": {\"reasoning\": {\"description\": \"Step-by-step reasoning\", \"title\": \"Reasoning\", \"type\": \"string\"}, \"final_answer\": {\"description\": \"Final numeric answer\", \"title\": \"Final Answer\", \"type\": \"number\"}}, \"required\": [\"reasoning\", \"final_answer\"]}\n",
+      "```\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain.output_parsers import PydanticOutputParser\n",
+    "\n",
+    "parser_gsm8k = PydanticOutputParser(pydantic_object=GSM8KOutput)\n",
+    "\n",
+    "prompt_gsm8k = PromptTemplate(\n",
+    "    input_variables=[\"question\"],\n",
+    "    partial_variables={\"format_instructions\": parser_gsm8k.get_format_instructions()},\n",
+    "    template=(\n",
+    "        \"\"\"You are a careful mathematician. Solve the problem step by step.\n",
+    "        Question: {question}\n",
+    "        {format_instructions}\"\"\"\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "print(parser_gsm8k.get_format_instructions())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d1dcc480",
+   "metadata": {},
+   "source": [
+    "**Consigne** : Modifier la classe `Benchmark` et la sous-classe `GMS8KBenchmark` pour intégrer ces évolutions."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "id": "542a31d6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.runnables import Runnable\n",
+    "from langchain_core.prompts import PromptTemplate\n",
+    "\n",
+    "\n",
+    "class Benchmark:\n",
+    "    name: str\n",
+    "\n",
+    "    def __init__(self, prompt: PromptTemplate, parser: PydanticOutputParser):\n",
+    "        self.prompt = prompt\n",
+    "        self.parser = parser\n",
+    "\n",
+    "    def load_data(self):\n",
+    "        raise NotImplementedError\n",
+    "\n",
+    "    def build_chain(self, model) -> Runnable:\n",
+    "        raise NotImplementedError\n",
+    "\n",
+    "    def get_reference(self, sample):\n",
+    "        raise NotImplementedError\n",
+    "\n",
+    "    def score(self, prediction, reference):\n",
+    "        raise NotImplementedError"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c94f1dd1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class GSM8KBenchmark(Benchmark):\n",
+    "    name = \"GSM8K\"\n",
+    "\n",
+    "    def load_data(self):\n",
+    "        return load_dataset(\"gsm8k\", \"main\", split=\"test\")\n",
+    "\n",
+    "    def build_chain(self, model):\n",
+    "        return {\"question\": RunnablePassthrough()} | self.prompt | model | self.parser\n",
+    "\n",
+    "    def get_reference(self, sample):\n",
+    "        match = search(r\"#### (\\d+)\", sample[\"answer\"])\n",
+    "        return float(match.group(1)) if match else None\n",
+    "\n",
+    "    def score(self, prediction: GSM8KOutput, reference: float | None):\n",
+    "        if reference is None:\n",
+    "            return 0.0\n",
+    "        return 1.0 if prediction.final_answer == reference else 0.0"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b2076f24",
+   "metadata": {},
+   "source": [
+    "**Consigne** : Utiliser la fonction `run_benchmark` et vérifier que tout fonctionne."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "id": "31e433b0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Running GSM8K: 100%|██████████| 5/5 [01:01<00:00, 12.25s/it]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'benchmark': 'GSM8K', 'model': 'gemma3:4b', 'num_samples': 5, 'accuracy': 0.8}"
+      ]
+     },
+     "execution_count": 69,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gsm8k = GSM8KBenchmark(\n",
+    "    prompt=prompt_gsm8k,\n",
+    "    parser=parser_gsm8k,\n",
+    ")\n",
+    "\n",
+    "run_benchmark(\"gemma3:4b\", gsm8k, max_samples=5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b7ed90cd",
+   "metadata": {},
+   "source": [
+    "**Consigne** : Réaliser la même modification pour HellaSwag, et vérifier que cela fonctionne."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e678bed2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class HellaSwagOutput(BaseModel):\n",
+    "    choice: int = Field(description=\"Index of the chosen continuation\")\n",
+    "\n",
+    "\n",
+    "class HellaSwagBenchmark(Benchmark):\n",
+    "    name = \"HellaSwag\"\n",
+    "\n",
+    "    def load_data(self):\n",
+    "        return load_dataset(\"hellaswag\", split=\"validation\")\n",
+    "\n",
+    "    def build_chain(self, model):\n",
+    "        return (\n",
+    "            {\n",
+    "                \"context\": lambda x: x[\"ctx\"],\n",
+    "                \"choices\": lambda x: \"\\n\".join(\n",
+    "                    f\"{index}: {choice}\" for index, choice in enumerate(x[\"endings\"])\n",
+    "                ),\n",
+    "            }\n",
+    "            | self.prompt\n",
+    "            | model\n",
+    "            | self.parser\n",
+    "        )\n",
+    "\n",
+    "    def get_reference(self, sample):\n",
+    "        return str(sample[\"label\"])\n",
+    "\n",
+    "    def score(self, prediction: HellaSwagOutput, reference: str) -> float:\n",
+    "        return 1.0 if str(prediction.choice) == reference else 0.0\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2455f816",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Running HellaSwag: 100%|██████████| 5/5 [00:15<00:00,  3.12s/it]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'benchmark': 'HellaSwag',\n",
+       " 'model': 'gemma3:4b',\n",
+       " 'num_samples': 5,\n",
+       " 'accuracy': 1.0}"
+      ]
+     },
+     "execution_count": 65,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "parser_hellaswag = PydanticOutputParser(pydantic_object=HellaSwagOutput)\n",
+    "\n",
+    "prompt_HellaSwag = PromptTemplate(\n",
+    "    input_variables=[\"context\", \"choices\"],\n",
+    "    partial_variables={\n",
+    "        \"format_instructions\": parser_hellaswag.get_format_instructions()\n",
+    "    },\n",
+    "    template=(\n",
+    "        \"\"\"You will be given a context and then different choices. You need to find the most likely continuation to the context.\n",
+    "        Context: {context}\n",
+    "        Choices: {choices}\n",
+    "        {format_instructions}\"\"\"\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "hella_swag = HellaSwagBenchmark(\n",
+    "    prompt=prompt_HellaSwag,\n",
+    "    parser=parser_hellaswag,\n",
+    ")\n",
+    "\n",
+    "run_benchmark(\"gemma3:4b\", hella_swag, max_samples=5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ba9acd54",
+   "metadata": {},
+   "source": [
+    "## Pour aller plus loin\n",
+    "\n",
+    "On pourrait implémenter d'autres benchmark, comparer vraiment des modèles entre eux, comparer des prompts entre eux..."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

M2/Generative AI/TP1/TP2 Benchmark - Starter.ipynb

@@ -1,456 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "172a7a9f",
-   "metadata": {},
-   "source": [
-    "# TP2 - Benchmark automatique\n",
-    "\n",
-    "Dans ce TP nous allons définir une fonction pour mesurer les performances d'un modèle de langage via l'exécution de plusieurs benchmarks. Nous avons vu en cours trois manières de mesurer la performance d'un modèle de langage qu'on peut résumer à:\n",
-    "1. **Évaluation automatique**: via un ensemble de questions dont on connait la réponse\n",
-    "2. **Évaluation humaine**: qualification humaine de la réponse d'un modèle à une question\n",
-    "3. **Évaluation par modèle de langage**: notation ou comparaison de réponse d'un ou plusieurs modèles par un autre modèle\n",
-    "\n",
-    "Nous nous intéressons ici au premier point, en particulier avec les benchmarks [GSM8K](https://huggingface.co/datasets/openai/gsm8k) et [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag).\n",
-    "Dans l'ensemble du notebook nous utiliserons la librairie LangChain.\n",
-    "\n",
-    "Il est à garder en tête que ce notebook n'a qu'une portée pédagogique et n'est pas forcément à jour puisque le domaine évolue rapidement, ni que les pratiques sont celles validées par l'industrie.\n",
-    "\n",
-    "## Uniformisation des benchmarks\n",
-    "\n",
-    "Pour chaque benchmark que l'on considère, nous avons besoin de plusieurs informations :\n",
-    "* **Dataset** : une fonction pour charger les questions du benchmark\n",
-    "* **Référence** : une fonction capable d'identifier la réponse attentue\n",
-    "* **Prompt** : un prompt qui permet de demander correctement au modèle de répondre à la question\n",
-    "* **Chaîne** : une fonction qui renvoie la chaîne de traitement de LangChain\n",
-    "* **Score** : une fonction qui score la performance d'un modèle sur une question\n",
-    "\n",
-    "Nous allons commencer par créer une classe qui regroupe ces desiderata :\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "cd75374d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain_core.runnables import Runnable\n",
-    "from langchain_core.prompts import PromptTemplate\n",
-    "\n",
-    "\n",
-    "class Benchmark:\n",
-    "    name: str\n",
-    "\n",
-    "    def __init__(self, prompt: PromptTemplate):\n",
-    "        self.prompt = prompt\n",
-    "\n",
-    "    def load_data(self):\n",
-    "        raise NotImplementedError\n",
-    "\n",
-    "    def build_chain(self, model) -> Runnable:\n",
-    "        raise NotImplementedError\n",
-    "\n",
-    "    def get_reference(self, sample):\n",
-    "        raise NotImplementedError\n",
-    "\n",
-    "    def score(self, prediction, reference):\n",
-    "        raise NotImplementedError"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e2ab41df",
-   "metadata": {},
-   "source": [
-    "Pour rendre cette classe plus concrète, commençons par travailler avec le benchmark [GSM8K](https://huggingface.co/datasets/openai/gsm8k).\n",
-    "\n",
-    "### Benchmark GSM8K\n",
-    "\n",
-    "On commence par charger le dataset et observer une question.\n",
-    "\n",
-    "**Consigne** : Résoudre la question *à la main* et vérifier votre réponse. On recommande d'explorer plusieurs questions."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "93979ba0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import numpy as np; np.random.seed(42)\n",
-    "from datasets import load_dataset\n",
-    "\n",
-    "dataset = load_dataset(\"gsm8k\", \"main\")\n",
-    "dataset = dataset[\"test\"]\n",
-    "\n",
-    "print(f\"Number of questions: {len(dataset)}\")\n",
-    "index = 0\n",
-    "print(\"Example of question:\\n\", dataset[index][\"question\"])\n",
-    "print(\"And its answer:\\n\", dataset[index][\"answer\"])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "82d797f0",
-   "metadata": {},
-   "source": [
-    "Après avoir inspecté plusieurs éléments du dataset, on remarque que la réponse finale est placée après la chaîne de caractères \"####\".\n",
-    "\n",
-    "**Consigne**: Construire une fonction `get_reference` qui prend en argument un élément de GMS8K (dictionnaire avec question et réponse) et renvoie la réponse attendue (string). On pourra utiliser la fonction [`search`](https://docs.python.org/3/library/re.html#re.search) de la librairie [`re`](https://docs.python.org/3/library/re.html#).\n",
-    "Puis tester cette fonction sur l'exemple précédent."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b336056a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4c137e6a",
-   "metadata": {},
-   "source": [
-    "Il nous reste maintenant à définir un prompt tel que l'on puisse appeler un modèle et tester notre mécanique."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "0b899872",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain_core.prompts import PromptTemplate\n",
-    "\n",
-    "prompt = PromptTemplate(\n",
-    "    input_variables=[\"question\"],\n",
-    "    template=(\n",
-    "        \"\"\"You are a careful mathematician. Solve the problem step by step, then display your answer in the end.\n",
-    "        Question: {question}\n",
-    "        Answer:\"\"\"\n",
-    "    )\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "36433b53",
-   "metadata": {},
-   "source": [
-    "En intégrant l'appel à un modèle via Ollama sur notre ordinateur, on peut définir avec LangChain :"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2f0676b6",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain_core.runnables import RunnablePassthrough\n",
-    "from langchain_core.output_parsers import StrOutputParser\n",
-    "from langchain_ollama import OllamaLLM\n",
-    "\n",
-    "model = OllamaLLM(model=\"gemma3:4b\")\n",
-    "\n",
-    "chain = (\n",
-    "    {\"question\": RunnablePassthrough()}\n",
-    "    | prompt\n",
-    "    | model\n",
-    "    | StrOutputParser()\n",
-    ")\n",
-    "\n",
-    "index = 0\n",
-    "\n",
-    "question = dataset[index][\"question\"]\n",
-    "answer = get_reference(dataset[index])\n",
-    "response = chain.invoke(question)\n",
-    "print(f\"Model answer : {response}\")\n",
-    "print(f\"The answer was : {answer}\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "97dd7db7",
-   "metadata": {},
-   "source": [
-    "Il nous faut extraire la dernière valeur numérique pour obtenir automatiquement la réponse du modèle.\n",
-    "\n",
-    "**Consigne** : Définir une fonction `score` qui prend en paramètre la réponse du modèle et la réponse attendue puis renvoie si les deux réponses sont identiques (1 / 0). On pourra utiliser la fonction [`findall`](https://docs.python.org/3/library/re.html#re.findall) de la librairie `re`.\n",
-    "Puis l'appliquer sur l'exemple précédent."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ad43cf84",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "a2ec5088",
-   "metadata": {},
-   "source": [
-    "Nous avons l'ensemble des éléments nécessaire pour définir la classe `GSM8KBenchmark` depuis la classe `Benchmark` que nous avons défini précédemment.\n",
-    "\n",
-    "**Consigne** : Définir cette classe comme sous-classe de `Benchmark`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d83f4394",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "dfc3cb78",
-   "metadata": {},
-   "source": [
-    "Il est maintenant temps de définir une fonction qui *fait* le benchmark.\n",
-    "\n",
-    "**Consigne** : Définir une fonction `run_benchmark` qui prend en paramètre :\n",
-    "* `model_name` : le nom du modèle Ollama que l'on veut tester\n",
-    "* `benchmark` : la classe benchmark que l'on souhaite tester\n",
-    "* `max_samples` : le nombre maximum de questions que l'on souhaite utiliser\n",
-    "\n",
-    "Puisque l'object avec lequel nous travaillons est un dataset HuggingFace, pour sélectionner $n$ lignes, on utilisera \n",
-    "```python\n",
-    "dataset = dataset.select(range(max_samples))\n",
-    "```\n",
-    "De cette manière on préserve la structure."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2d7125af",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "81de8940",
-   "metadata": {},
-   "source": [
-    "**Consigne** : Utiliser la fonction `run_benchmark` en définissant un prompt pour GSM8K."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f6bbeb53",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0c943124",
-   "metadata": {},
-   "source": [
-    "### HellaSwag\n",
-    "\n",
-    "Maintenant que nous avons réussi à le faire pour le dataset GMS8K, attaquons-nous à [HellaSwag](https://huggingface.co/datasets/Rowan/hellaswag).\n",
-    "\n",
-    "**Consigne** : En suivant la même approche que précédemment, implémenter une sous classe `HellaSwagBenchmark` à partir de la classe `Benchmark`. Puis utiliser la fonction `run_benchmark` pour valider votre travail."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "32886901",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "96a3031a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "c542783c",
-   "metadata": {},
-   "source": [
-    "## Réponses structurées\n",
-    "\n",
-    "Sur quelques exemples tout semble fonctionner ! Mais il y a au moins une fragilité dans notre travail : la récupération de la réponse est peu fiable et largement dépendante des prompts.\n",
-    "\n",
-    "\n",
-    "Par exemple pour GMS8K, on aimerait avoir une réponse sous la forme d'un JSON :\n",
-    "```json\n",
-    "{\n",
-    "  \"reasoning\": \"étapes de raisonnement\",\n",
-    "  \"final_answer\": 18\n",
-    "}\n",
-    "```\n",
-    "\n",
-    "De cette manière ce serait particulièrement simple d'extraire la réponse, sans pour autant ne pas avoir de *réflexion* du modèle. En revanche pour HellaSwag, un JSON extrêment simple suffit :\n",
-    "```json\n",
-    "{\n",
-    "    \"choice\": 2\n",
-    "}\n",
-    "```\n",
-    "\n",
-    "Pour forcer le modèle à suivre ces formats, nous allons utiliser l'option [Pydantic](https://docs.langchain.com/oss/python/langchain/structured-output). Elle s'utilise comme suit, pour GSM8K :"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "988dbca3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from pydantic import BaseModel, Field\n",
-    "\n",
-    "class GSM8KOutput(BaseModel):\n",
-    "    reasoning: str = Field(description=\"Step-by-step reasoning\")\n",
-    "    final_answer: float = Field(description=\"Final numeric answer\")\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d855adfe",
-   "metadata": {},
-   "source": [
-    "Concernant l'intégration dans le prompt :"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f25afddc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.output_parsers import PydanticOutputParser\n",
-    "\n",
-    "parser_gsm8k = PydanticOutputParser(pydantic_object=GSM8KOutput)\n",
-    "\n",
-    "prompt_gsm8k = PromptTemplate(\n",
-    "    input_variables=[\"question\"],\n",
-    "    partial_variables={\"format_instructions\": parser_gsm8k.get_format_instructions()},\n",
-    "    template=(\n",
-    "        \"\"\"You are a careful mathematician. Solve the problem step by step.\n",
-    "        Question: {question}\n",
-    "        {format_instructions}\"\"\"\n",
-    "    ),\n",
-    ")\n",
-    "\n",
-    "print(parser_gsm8k.get_format_instructions())"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "d1dcc480",
-   "metadata": {},
-   "source": [
-    "**Consigne** : Modifier la classe `Benchmark` et la sous-classe `GMS8KBenchmark` pour intégrer ces évolutions."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "542a31d6",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c94f1dd1",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b2076f24",
-   "metadata": {},
-   "source": [
-    "**Consigne** : Utiliser la fonction `run_benchmark` et vérifier que tout fonctionne."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "31e433b0",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "b7ed90cd",
-   "metadata": {},
-   "source": [
-    "**Consigne** : Réaliser la même modification pour HellaSwag, et vérifier que cela fonctionne."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e678bed2",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2455f816",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "markdown",
-   "id": "ba9acd54",
-   "metadata": {},
-   "source": [
-    "## Pour aller plus loin\n",
-    "\n",
-    "On pourrait implémenter d'autres benchmark, comparer vraiment des modèles entre eux, comparer des prompts entre eux..."
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.13"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

M2/Statistiques Non Paramétrique/TP2.Rmd

@@ -0,0 +1,199 @@
+# Estimateur de Nadaraya-Watson
+
+1. 
+```{r}
+n = 500
+sigma2 = 0.01
+epsilon = rnorm(n, 0, sqrt(sigma2))
+
+X = runif(n, 0, 1)
+Y = exp(abs(X - 0.5)) + epsilon
+
+plot(X, Y, pch = 3)
+```
+
+2. 
+
+```{r}
+library(np)
+
+bw = npregbw(Y ~ X)
+ghat = npreg(bw)
+x = seq(0, 1, length.out = 100)
+
+plot(ghat)
+points(x, exp(abs(x - 0.5)), type = 'l', col = 'darkgreen')
+legend(
+  'top',
+  c(expression(hat(g)[h]), 'g'),
+  col = c(1, 'darkgreen'),
+  lty = rep(1, 2)
+)
+```
+
+3.  
+
+...
+
+# Estimation par projection
+
+4. 
+
+```{r}
+mmax = 100
+Dmax = 2 * mmax + 1
+phiX = matrix(1, n, Dmax)
+for (j in 1:mmax) {
+  phiX[, 2 * j] = sqrt(2) * cos(2 * pi * j * X)
+  phiX[, 2 * j + 1] = sqrt(2) * sin(2 * pi * j * X)
+}
+G = crossprod(phiX) / n
+```
+
+5. 
+
+```{r}
+hatb = crossprod(phiX, Y) / n
+hata = solve(G, hatb)
+```
+
+6. 
+
+```{r}
+p = 500
+t = seq(0, 1, length.out = p)
+phi = matrix(1, Dmax, p)
+
+for (j in 1:mmax) {
+  phi[2 * j, ] = sqrt(2) * cos(2 * pi * j * t)
+  phi[2 * j + 1, ] = sqrt(2) * sin(2 * pi * j * t)
+}
+
+hatg = crossprod(phi, hata)
+
+plot(t, hatg, type = 'l', col = 'red')
+points(t, exp(abs(t - 0.5)), type = 'l', col = 'darkgreen')
+legend(
+  'topright',
+  c('g', expression(hat(g)[501])),
+  lty = c(1, 1),
+  col = c('darkgreen', 'red')
+)
+```
+
+7. 
+
+
+```{r}
+hata1 = solve(G[1, 1], hatb[1])
+hatg1 = hata1 * phi[1, ]
+
+hata3 = solve(G[1:3, 1:3], hatb[1:3])
+hatg3 = crossprod(phi[1:3, ], hata3)
+
+hata5 = solve(G[1:5, 1:5], hatb[1:5])
+hatg5 = crossprod(phi[1:5, ], hata5)
+
+plot(t, hatg, type = 'l', col = 'red', main = 'Projection estimators')
+points(t, exp(abs(t - 0.5)), type = 'l', col = 'darkgreen')
+points(t, hatg1, type = 'l', col = 'blue')
+points(t, hatg3, type = 'l', col = 'green')
+points(t, hatg5, type = 'l', col = 'purple')
+legend(
+  'topright',
+  c(
+    'g',
+    expression(hat(g)[1]),
+    expression(hat(g)[3]),
+    expression(hat(g)[5]),
+    expression(hat(g)[501])
+  ),
+  lty = rep(1, 5),
+  col = c('darkgreen', 'blue', 'green', 'purple', 'red')
+)
+```
+
+```{r}
+kappa = 2.5
+mmax = 10
+gamman = rep(NA, mmax)
+penm = sigma2 * (2 * (1:mmax) + 1) / n
+for (m in 1:mmax) {
+  hatam = solve(G[1:(2 * m + 1), 1:(2 * m + 1)], hatb[1:(2 * m + 1)])
+  hatgmX = crossprod(t(phiX[, 1:(2 * m + 1)]), hatam)
+  gamman[m] = mean((Y - hatgmX)^2)
+}
+crit = gamman + kappa * penm
+plot(
+  (2 * (1:mmax) + 1),
+  gamman,
+  type = 'l',
+  ylim = range(c(gamman, crit, kappa * penm)),
+  xlab = 'm',
+  ylab = 'crit(m)',
+  col = 'blue'
+)
+points((2 * (1:mmax) + 1), kappa * penm, type = 'l', col = 'red')
+points((2 * (1:mmax) + 1), crit, type = 'l')
+legend(
+  'left',
+  c('contraste', 'penalité', 'critère'),
+  col = c('blue', 'red', 1),
+  lty = rep(1, 3)
+)
+hatm = which.min(crit)
+points(2 * hatm + 1, crit[hatm], pch = 16)
+```
+
+```{r}
+hatahatm = solve(G[1:(2 * hatm + 1), 1:(2 * hatm + 1)], hatb[1:(2 * hatm + 1)])
+hatghatm = crossprod(phi[1:(2 * hatm + 1), ], hatahatm)
+plot(
+  t,
+  hatghatm,
+  type = 'l',
+  col = 'green',
+  main = 'Selected projection estimators',
+  ylab = expression(hat(g)[hat(m)](t))
+)
+points(t, exp(abs(t - 0.5)), type = 'l', col = 'darkgreen')
+```
+
+# Application aux données réelles
+
+9. 
+
+
+```{r}
+data(cps71)
+attach(cps71)
+plot(age, logwage, xlab = "Age", ylab = "log(wage)")
+```
+
+
+```{r}
+bw = npregbw(logwage ~ age)
+ghat = npreg(bw)
+
+print(bw)
+
+plot(ghat, xlab = "Age", ylab = "log(wage)")
+points(age, logwage)
+```
+
+10. 
+
+```{r}
+data(Italy)
+attach(Italy)
+
+plot(year, gdp, xlab = "Année", ylab = "GDP")
+```
+
+
+```{r}
+bw = npregbw(gdp ~ year)
+ghat = npreg(bw)
+
+plot(ghat, xlab="Année", ylab="gdp",type='l')
+```

pyproject.toml

@@ -5,29 +5,37 @@ description = "A curated collection of mathematics and data science projects dev
 readme = "README.md"
 requires-python = ">= 3.11"
 dependencies = [
+    "accelerate>=1.12.0",
     "catboost>=1.2.8",
+    "datasets>=4.4.2",
     "faiss-cpu>=1.13.2",
+    "gymnasium[toy-text]>=1.2.3",
     "imblearn>=0.0",
     "ipykernel>=6.29.5",
     "langchain>=1.2.0",
     "langchain-community>=0.4.1",
     "langchain-huggingface>=1.2.0",
     "langchain-ollama>=1.0.1",
+    "langchain-text-splitters>=1.1.0",
     "matplotlib>=3.10.1",
     "nbformat>=5.10.4",
     "numpy>=2.2.5",
     "opencv-python>=4.11.0.86",
+    "openpyxl>=3.1.5",
     "pandas>=2.2.3",
     "pandas-stubs>=2.3.2.250926",
     "plotly>=6.3.0",
+    "polars>=1.37.0",
     "pypdf>=6.5.0",
     "scikit-learn>=1.6.1",
     "scipy>=1.15.2",
     "seaborn>=0.13.2",
     "sentence-transformers>=5.2.0",
     "shap>=0.49.1",
+    "statsmodels>=0.14.6",
     "tensorflow>=2.20.0",
     "tf-keras>=2.20.1",
+    "umap-learn>=0.5.11",
     "xgboost>=3.1.2",
     "yfinance>=0.2.66",
 ]
@@ -56,7 +64,8 @@ ignore = [
     "T201", # Print
     "N806",
     "N803",
-    "N802"
+    "N802",
+    "N816",
 ]
 
 # Exclure certains fichiers ou répertoires

uv.lock

@@ -22,6 +22,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" },
 ]
 
+[[package]]
+name = "accelerate"
+version = "1.12.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "huggingface-hub" },
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "psutil" },
+    { name = "pyyaml" },
+    { name = "safetensors" },
+    { name = "torch" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4a/8e/ac2a9566747a93f8be36ee08532eb0160558b07630a081a6056a9f89bf1d/accelerate-1.12.0.tar.gz", hash = "sha256:70988c352feb481887077d2ab845125024b2a137a5090d6d7a32b57d03a45df6", size = 398399, upload-time = "2025-11-21T11:27:46.973Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9f/d2/c581486aa6c4fbd7394c23c47b83fa1a919d34194e16944241daf9e762dd/accelerate-1.12.0-py3-none-any.whl", hash = "sha256:3e2091cd341423207e2f084a6654b1efcd250dc326f2a37d6dde446e07cabb11", size = 380935, upload-time = "2025-11-21T11:27:44.522Z" },
+]
+
 [[package]]
 name = "aiohappyeyeballs"
 version = "2.6.1"
@@ -484,6 +502,31 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" },
 ]
 
+[[package]]
+name = "datasets"
+version = "4.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "dill" },
+    { name = "filelock" },
+    { name = "fsspec", extra = ["http"] },
+    { name = "httpx" },
+    { name = "huggingface-hub" },
+    { name = "multiprocess" },
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "pandas" },
+    { name = "pyarrow" },
+    { name = "pyyaml" },
+    { name = "requests" },
+    { name = "tqdm" },
+    { name = "xxhash" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c4/54/9359803da96bc65439a28fbb014dc2c90b7d4d8034a93b72362b0d40191f/datasets-4.4.2.tar.gz", hash = "sha256:9de16e415c4ba4713eac0493f7c7dc74f3aa21599297f00cc6ddab409cb7b24b", size = 586474, upload-time = "2025-12-19T15:03:09.129Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7b/b5/fefa518c809de7bced5cddb7c21c010da66fa2ae494bda96844a280cc6ce/datasets-4.4.2-py3-none-any.whl", hash = "sha256:6f5ef3417504d9cd663c71c1b90b9a494ff4c2076a2cd6a6e40ceee6ad95befc", size = 512268, upload-time = "2025-12-19T15:03:07.087Z" },
+]
+
 [[package]]
 name = "debugpy"
 version = "1.8.14"
@@ -514,6 +557,24 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/4e/8c/f3147f5c4b73e7550fe5f9352eaa956ae838d5c51eb58e7a25b9f3e2643b/decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a", size = 9190, upload-time = "2025-02-24T04:41:32.565Z" },
 ]
 
+[[package]]
+name = "dill"
+version = "0.4.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" },
+]
+
+[[package]]
+name = "et-xmlfile"
+version = "2.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d3/38/af70d7ab1ae9d4da450eeec1fa3918940a5fafb9055e934af8d6eb0c2313/et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54", size = 17234, upload-time = "2024-10-25T17:25:40.039Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" },
+]
+
 [[package]]
 name = "executing"
 version = "2.2.0"
@@ -548,6 +609,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/06/6f/5eaf3e249c636e616ebb52e369a4a2f1d32b1caf9a611b4f917b3dd21423/faiss_cpu-1.13.2-cp314-cp314-win_arm64.whl", hash = "sha256:8113a2a80b59fe5653cf66f5c0f18be0a691825601a52a614c30beb1fca9bc7c", size = 8556374, upload-time = "2025-12-24T10:27:36.653Z" },
 ]
 
+[[package]]
+name = "farama-notifications"
+version = "0.0.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2e/2c/8384832b7a6b1fd6ba95bbdcae26e7137bb3eedc955c42fd5cdcc086cfbf/Farama-Notifications-0.0.4.tar.gz", hash = "sha256:13fceff2d14314cf80703c8266462ebf3733c7d165336eee998fc58e545efd18", size = 2131, upload-time = "2023-02-27T18:28:41.047Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/05/2c/ffc08c54c05cdce6fbed2aeebc46348dbe180c6d2c541c7af7ba0aa5f5f8/Farama_Notifications-0.0.4-py3-none-any.whl", hash = "sha256:14de931035a41961f7c056361dc7f980762a143d05791ef5794a751a2caf05ae", size = 2511, upload-time = "2023-02-27T18:28:39.447Z" },
+]
+
 [[package]]
 name = "fastjsonschema"
 version = "2.21.2"
@@ -726,11 +796,16 @@ wheels = [
 
 [[package]]
 name = "fsspec"
-version = "2025.12.0"
+version = "2025.10.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/b6/27/954057b0d1f53f086f681755207dda6de6c660ce133c829158e8e8fe7895/fsspec-2025.12.0.tar.gz", hash = "sha256:c505de011584597b1060ff778bb664c1bc022e87921b0e4f10cc9c44f9635973", size = 309748, upload-time = "2025-12-03T15:23:42.687Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b", size = 201422, upload-time = "2025-12-03T15:23:41.434Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" },
+]
+
+[package.optional-dependencies]
+http = [
+    { name = "aiohttp" },
 ]
 
 [[package]]
@@ -848,6 +923,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/34/80/de3eb55eb581815342d097214bed4c59e806b05f1b3110df03b2280d6dfd/grpcio-1.74.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24", size = 4489214, upload-time = "2025-07-24T18:53:59.771Z" },
 ]
 
+[[package]]
+name = "gymnasium"
+version = "1.2.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cloudpickle" },
+    { name = "farama-notifications" },
+    { name = "numpy" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/76/59/653a9417d98ed3e29ef9734ba52c3495f6c6823b8d5c0c75369f25111708/gymnasium-1.2.3.tar.gz", hash = "sha256:2b2cb5b5fbbbdf3afb9f38ca952cc48aa6aa3e26561400d940747fda3ad42509", size = 829230, upload-time = "2025-12-18T16:51:10.234Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/56/d3/ea5f088e3638dbab12e5c20d6559d5b3bdaeaa1f2af74e526e6815836285/gymnasium-1.2.3-py3-none-any.whl", hash = "sha256:e6314bba8f549c7fdcc8677f7cd786b64908af6e79b57ddaa5ce1825bffb5373", size = 952113, upload-time = "2025-12-18T16:51:08.445Z" },
+]
+
+[package.optional-dependencies]
+toy-text = [
+    { name = "pygame" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -1795,6 +1890,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" },
 ]
 
+[[package]]
+name = "multiprocess"
+version = "0.70.18"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "dill" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" },
+    { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" },
+    { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917, upload-time = "2025-04-17T03:11:24.044Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" },
+]
+
 [[package]]
 name = "multitasking"
 version = "0.0.12"
@@ -2101,6 +2216,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a4/7d/f1c30a92854540bf789e9cd5dde7ef49bbe63f855b85a2e6b3db8135c591/opencv_python-4.11.0.86-cp37-abi3-win_amd64.whl", hash = "sha256:085ad9b77c18853ea66283e98affefe2de8cc4c1f43eda4c100cf9b2721142ec", size = 39488044, upload-time = "2025-01-16T13:52:21.928Z" },
 ]
 
+[[package]]
+name = "openpyxl"
+version = "3.1.5"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "et-xmlfile" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3d/f9/88d94a75de065ea32619465d2f77b29a0469500e99012523b91cc4141cd1/openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050", size = 186464, upload-time = "2024-06-28T14:03:44.161Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" },
+]
+
 [[package]]
 name = "opt-einsum"
 version = "3.4.0"
@@ -2374,6 +2501,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650, upload-time = "2024-04-05T09:43:53.299Z" },
 ]
 
+[[package]]
+name = "patsy"
+version = "1.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/be/44/ed13eccdd0519eff265f44b670d46fbb0ec813e2274932dc1c0e48520f7d/patsy-1.0.2.tar.gz", hash = "sha256:cdc995455f6233e90e22de72c37fcadb344e7586fb83f06696f54d92f8ce74c0", size = 399942, upload-time = "2025-10-20T16:17:37.535Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f1/70/ba4b949bdc0490ab78d545459acd7702b211dfccf7eb89bbc1060f52818d/patsy-1.0.2-py2.py3-none-any.whl", hash = "sha256:37bfddbc58fcf0362febb5f54f10743f8b21dd2aa73dec7e7ef59d1b02ae668a", size = 233301, upload-time = "2025-10-20T16:17:36.563Z" },
+]
+
 [[package]]
 name = "peewee"
 version = "3.18.2"
@@ -2473,6 +2612,34 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/95/a9/12e2dc726ba1ba775a2c6922d5d5b4488ad60bdab0888c337c194c8e6de8/plotly-6.3.0-py3-none-any.whl", hash = "sha256:7ad806edce9d3cdd882eaebaf97c0c9e252043ed1ed3d382c3e3520ec07806d4", size = 9791257, upload-time = "2025-08-12T20:22:09.205Z" },
 ]
 
+[[package]]
+name = "polars"
+version = "1.37.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "polars-runtime-32" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/6c/b5/ce40267c54b66f93572d84f7ba1c216b72a71cb2235e3724fab0911541fe/polars-1.37.0.tar.gz", hash = "sha256:6bbbeefb6f02f848d46ad4f4e922a92573986fd38611801c696bae98b02be4c8", size = 715429, upload-time = "2026-01-10T12:28:06.741Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/31/07/d890382bbfdeb25db039ef4a8c8f93b3faf0016e18130513274204954203/polars-1.37.0-py3-none-any.whl", hash = "sha256:fcc549b9923ef1bd6fd99b5fd0a00dfedf85406f4758ae018a69bcd18a91f113", size = 805614, upload-time = "2026-01-10T12:26:47.897Z" },
+]
+
+[[package]]
+name = "polars-runtime-32"
+version = "1.37.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/30/92/b818590a5ebcc55657f5483f26133174bd2b9ca88457b60c93669a9d0c75/polars_runtime_32-1.37.0.tar.gz", hash = "sha256:954ddb056e3a2db2cbcaae501225ac5604d1599b6debd9c6dbdf8efbac0e6511", size = 2820371, upload-time = "2026-01-10T12:28:08.195Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f0/67/76162c9fcc71b917bdfd2804eaf0ab7cdb264a89b89af4f195a918f9f97d/polars_runtime_32-1.37.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3591f4b8e734126d713a12869d3727360acbbcd1d440b45d830497a317a5a8b3", size = 43518436, upload-time = "2026-01-10T12:26:51.442Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/ec/56f328e8fa4ebea453f5bc10c579774dff774a873ff224b3108d53c514f9/polars_runtime_32-1.37.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:47849420859159681e94589daad3a04ff66a2379c116ccd812d043f7ffe0094c", size = 39663939, upload-time = "2026-01-10T12:26:54.664Z" },
+    { url = "https://files.pythonhosted.org/packages/4c/b2/f1ea0edba327a92ce0158b7a0e4abe21f541e44c9fb8ec932cc47592ca5c/polars_runtime_32-1.37.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4648ea1e821b9a841b2a562f27bcf54ff1ad21f9c217adcf0f7d0b3c33dc6400", size = 41481348, upload-time = "2026-01-10T12:26:57.598Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/21/788a3dd724bb21cf42e2f4daa6510a47787e8b30dd535aa6cae20ea968d0/polars_runtime_32-1.37.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5272b6f1680a3e0d77c9f07cb5a54f307079eb5d519c71aa3c37b9af0ee03a9e", size = 45168069, upload-time = "2026-01-10T12:27:00.98Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/73/823d6534a20ebdcec4b7706ab2b3f2cfb8e07571305f4e7381cc22d83e31/polars_runtime_32-1.37.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:73301ef4fe80d8d748085259a4063ac52ff058088daa702e2a75e7d1ab7f14fc", size = 41675645, upload-time = "2026-01-10T12:27:04.334Z" },
+    { url = "https://files.pythonhosted.org/packages/30/54/1bacad96dc2b67d33b886a45b249777212782561493718785cb27c7c362a/polars_runtime_32-1.37.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:c60d523d738a7b3660d9abdfaff798f7602488f469d427865965b0bd2e40473a", size = 44737715, upload-time = "2026-01-10T12:27:08.152Z" },
+    { url = "https://files.pythonhosted.org/packages/38/e3/aad525d8d89b903fcfa2bd0b4cb66b8a6e83e80b3d1348c5a428092d2983/polars_runtime_32-1.37.0-cp310-abi3-win_amd64.whl", hash = "sha256:f87f76f16e8030d277ecca0c0976aca62ec2b6ba2099ee9c6f75dfc97e7dc1b1", size = 45018403, upload-time = "2026-01-10T12:27:11.292Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/4d/ddcaa5f2e18763e02e66d0fd2efca049a42fe96fbeda188e89aeb38dd6fa/polars_runtime_32-1.37.0-cp310-abi3-win_arm64.whl", hash = "sha256:7ffbd9487e3668b0a57519f7ab5ab53ab656086db9f62dceaab41393a07be721", size = 41026243, upload-time = "2026-01-10T12:27:14.563Z" },
+]
+
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.51"
@@ -2631,6 +2798,56 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" },
 ]
 
+[[package]]
+name = "pyarrow"
+version = "22.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/30/53/04a7fdc63e6056116c9ddc8b43bc28c12cdd181b85cbeadb79278475f3ae/pyarrow-22.0.0.tar.gz", hash = "sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9", size = 1151151, upload-time = "2025-10-24T12:30:00.762Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/2e/b7/18f611a8cdc43417f9394a3ccd3eace2f32183c08b9eddc3d17681819f37/pyarrow-22.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:3e294c5eadfb93d78b0763e859a0c16d4051fc1c5231ae8956d61cb0b5666f5a", size = 34272022, upload-time = "2025-10-24T10:04:28.973Z" },
+    { url = "https://files.pythonhosted.org/packages/26/5c/f259e2526c67eb4b9e511741b19870a02363a47a35edbebc55c3178db22d/pyarrow-22.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:69763ab2445f632d90b504a815a2a033f74332997052b721002298ed6de40f2e", size = 35995834, upload-time = "2025-10-24T10:04:35.467Z" },
+    { url = "https://files.pythonhosted.org/packages/50/8d/281f0f9b9376d4b7f146913b26fac0aa2829cd1ee7e997f53a27411bbb92/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b41f37cabfe2463232684de44bad753d6be08a7a072f6a83447eeaf0e4d2a215", size = 45030348, upload-time = "2025-10-24T10:04:43.366Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/e5/53c0a1c428f0976bf22f513d79c73000926cb00b9c138d8e02daf2102e18/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:35ad0f0378c9359b3f297299c3309778bb03b8612f987399a0333a560b43862d", size = 47699480, upload-time = "2025-10-24T10:04:51.486Z" },
+    { url = "https://files.pythonhosted.org/packages/95/e1/9dbe4c465c3365959d183e6345d0a8d1dc5b02ca3f8db4760b3bc834cf25/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8382ad21458075c2e66a82a29d650f963ce51c7708c7c0ff313a8c206c4fd5e8", size = 48011148, upload-time = "2025-10-24T10:04:59.585Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/b4/7caf5d21930061444c3cf4fa7535c82faf5263e22ce43af7c2759ceb5b8b/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1a812a5b727bc09c3d7ea072c4eebf657c2f7066155506ba31ebf4792f88f016", size = 50276964, upload-time = "2025-10-24T10:05:08.175Z" },
+    { url = "https://files.pythonhosted.org/packages/ae/f3/cec89bd99fa3abf826f14d4e53d3d11340ce6f6af4d14bdcd54cd83b6576/pyarrow-22.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ec5d40dd494882704fb876c16fa7261a69791e784ae34e6b5992e977bd2e238c", size = 28106517, upload-time = "2025-10-24T10:05:14.314Z" },
+    { url = "https://files.pythonhosted.org/packages/af/63/ba23862d69652f85b615ca14ad14f3bcfc5bf1b99ef3f0cd04ff93fdad5a/pyarrow-22.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d", size = 34211578, upload-time = "2025-10-24T10:05:21.583Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/d0/f9ad86fe809efd2bcc8be32032fa72e8b0d112b01ae56a053006376c5930/pyarrow-22.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8", size = 35989906, upload-time = "2025-10-24T10:05:29.485Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/a8/f910afcb14630e64d673f15904ec27dd31f1e009b77033c365c84e8c1e1d/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5", size = 45021677, upload-time = "2025-10-24T10:05:38.274Z" },
+    { url = "https://files.pythonhosted.org/packages/13/95/aec81f781c75cd10554dc17a25849c720d54feafb6f7847690478dcf5ef8/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe", size = 47726315, upload-time = "2025-10-24T10:05:47.314Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/d4/74ac9f7a54cfde12ee42734ea25d5a3c9a45db78f9def949307a92720d37/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e", size = 47990906, upload-time = "2025-10-24T10:05:58.254Z" },
+    { url = "https://files.pythonhosted.org/packages/2e/71/fedf2499bf7a95062eafc989ace56572f3343432570e1c54e6599d5b88da/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9", size = 50306783, upload-time = "2025-10-24T10:06:08.08Z" },
+    { url = "https://files.pythonhosted.org/packages/68/ed/b202abd5a5b78f519722f3d29063dda03c114711093c1995a33b8e2e0f4b/pyarrow-22.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d", size = 27972883, upload-time = "2025-10-24T10:06:14.204Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/d6/d0fac16a2963002fc22c8fa75180a838737203d558f0ed3b564c4a54eef5/pyarrow-22.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a", size = 34204629, upload-time = "2025-10-24T10:06:20.274Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/9c/1d6357347fbae062ad3f17082f9ebc29cc733321e892c0d2085f42a2212b/pyarrow-22.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901", size = 35985783, upload-time = "2025-10-24T10:06:27.301Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/c0/782344c2ce58afbea010150df07e3a2f5fdad299cd631697ae7bd3bac6e3/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691", size = 45020999, upload-time = "2025-10-24T10:06:35.387Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/8b/5362443737a5307a7b67c1017c42cd104213189b4970bf607e05faf9c525/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e0a15757fccb38c410947df156f9749ae4a3c89b2393741a50521f39a8cf202a", size = 47724601, upload-time = "2025-10-24T10:06:43.551Z" },
+    { url = "https://files.pythonhosted.org/packages/69/4d/76e567a4fc2e190ee6072967cb4672b7d9249ac59ae65af2d7e3047afa3b/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cedb9dd9358e4ea1d9bce3665ce0797f6adf97ff142c8e25b46ba9cdd508e9b6", size = 48001050, upload-time = "2025-10-24T10:06:52.284Z" },
+    { url = "https://files.pythonhosted.org/packages/01/5e/5653f0535d2a1aef8223cee9d92944cb6bccfee5cf1cd3f462d7cb022790/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:252be4a05f9d9185bb8c18e83764ebcfea7185076c07a7a662253af3a8c07941", size = 50307877, upload-time = "2025-10-24T10:07:02.405Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/f8/1d0bd75bf9328a3b826e24a16e5517cd7f9fbf8d34a3184a4566ef5a7f29/pyarrow-22.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:a4893d31e5ef780b6edcaf63122df0f8d321088bb0dee4c8c06eccb1ca28d145", size = 27977099, upload-time = "2025-10-24T10:08:07.259Z" },
+    { url = "https://files.pythonhosted.org/packages/90/81/db56870c997805bf2b0f6eeeb2d68458bf4654652dccdcf1bf7a42d80903/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:f7fe3dbe871294ba70d789be16b6e7e52b418311e166e0e3cba9522f0f437fb1", size = 34336685, upload-time = "2025-10-24T10:07:11.47Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/98/0727947f199aba8a120f47dfc229eeb05df15bcd7a6f1b669e9f882afc58/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ba95112d15fd4f1105fb2402c4eab9068f0554435e9b7085924bcfaac2cc306f", size = 36032158, upload-time = "2025-10-24T10:07:18.626Z" },
+    { url = "https://files.pythonhosted.org/packages/96/b4/9babdef9c01720a0785945c7cf550e4acd0ebcd7bdd2e6f0aa7981fa85e2/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c064e28361c05d72eed8e744c9605cbd6d2bb7481a511c74071fd9b24bc65d7d", size = 44892060, upload-time = "2025-10-24T10:07:26.002Z" },
+    { url = "https://files.pythonhosted.org/packages/f8/ca/2f8804edd6279f78a37062d813de3f16f29183874447ef6d1aadbb4efa0f/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6f9762274496c244d951c819348afbcf212714902742225f649cf02823a6a10f", size = 47504395, upload-time = "2025-10-24T10:07:34.09Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/f0/77aa5198fd3943682b2e4faaf179a674f0edea0d55d326d83cb2277d9363/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a9d9ffdc2ab696f6b15b4d1f7cec6658e1d788124418cb30030afbae31c64746", size = 48066216, upload-time = "2025-10-24T10:07:43.528Z" },
+    { url = "https://files.pythonhosted.org/packages/79/87/a1937b6e78b2aff18b706d738c9e46ade5bfcf11b294e39c87706a0089ac/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ec1a15968a9d80da01e1d30349b2b0d7cc91e96588ee324ce1b5228175043e95", size = 50288552, upload-time = "2025-10-24T10:07:53.519Z" },
+    { url = "https://files.pythonhosted.org/packages/60/ae/b5a5811e11f25788ccfdaa8f26b6791c9807119dffcf80514505527c384c/pyarrow-22.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bba208d9c7decf9961998edf5c65e3ea4355d5818dd6cd0f6809bec1afb951cc", size = 28262504, upload-time = "2025-10-24T10:08:00.932Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/b0/0fa4d28a8edb42b0a7144edd20befd04173ac79819547216f8a9f36f9e50/pyarrow-22.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9bddc2cade6561f6820d4cd73f99a0243532ad506bc510a75a5a65a522b2d74d", size = 34224062, upload-time = "2025-10-24T10:08:14.101Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/a8/7a719076b3c1be0acef56a07220c586f25cd24de0e3f3102b438d18ae5df/pyarrow-22.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e70ff90c64419709d38c8932ea9fe1cc98415c4f87ea8da81719e43f02534bc9", size = 35990057, upload-time = "2025-10-24T10:08:21.842Z" },
+    { url = "https://files.pythonhosted.org/packages/89/3c/359ed54c93b47fb6fe30ed16cdf50e3f0e8b9ccfb11b86218c3619ae50a8/pyarrow-22.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:92843c305330aa94a36e706c16209cd4df274693e777ca47112617db7d0ef3d7", size = 45068002, upload-time = "2025-10-24T10:08:29.034Z" },
+    { url = "https://files.pythonhosted.org/packages/55/fc/4945896cc8638536ee787a3bd6ce7cec8ec9acf452d78ec39ab328efa0a1/pyarrow-22.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6dda1ddac033d27421c20d7a7943eec60be44e0db4e079f33cc5af3b8280ccde", size = 47737765, upload-time = "2025-10-24T10:08:38.559Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/5e/7cb7edeb2abfaa1f79b5d5eb89432356155c8426f75d3753cbcb9592c0fd/pyarrow-22.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:84378110dd9a6c06323b41b56e129c504d157d1a983ce8f5443761eb5256bafc", size = 48048139, upload-time = "2025-10-24T10:08:46.784Z" },
+    { url = "https://files.pythonhosted.org/packages/88/c6/546baa7c48185f5e9d6e59277c4b19f30f48c94d9dd938c2a80d4d6b067c/pyarrow-22.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:854794239111d2b88b40b6ef92aa478024d1e5074f364033e73e21e3f76b25e0", size = 50314244, upload-time = "2025-10-24T10:08:55.771Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/79/755ff2d145aafec8d347bf18f95e4e81c00127f06d080135dfc86aea417c/pyarrow-22.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:b883fe6fd85adad7932b3271c38ac289c65b7337c2c132e9569f9d3940620730", size = 28757501, upload-time = "2025-10-24T10:09:59.891Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/d2/237d75ac28ced3147912954e3c1a174df43a95f4f88e467809118a8165e0/pyarrow-22.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7a820d8ae11facf32585507c11f04e3f38343c1e784c9b5a8b1da5c930547fe2", size = 34355506, upload-time = "2025-10-24T10:09:02.953Z" },
+    { url = "https://files.pythonhosted.org/packages/1e/2c/733dfffe6d3069740f98e57ff81007809067d68626c5faef293434d11bd6/pyarrow-22.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:c6ec3675d98915bf1ec8b3c7986422682f7232ea76cad276f4c8abd5b7319b70", size = 36047312, upload-time = "2025-10-24T10:09:10.334Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/2b/29d6e3782dc1f299727462c1543af357a0f2c1d3c160ce199950d9ca51eb/pyarrow-22.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3e739edd001b04f654b166204fc7a9de896cf6007eaff33409ee9e50ceaff754", size = 45081609, upload-time = "2025-10-24T10:09:18.61Z" },
+    { url = "https://files.pythonhosted.org/packages/8d/42/aa9355ecc05997915af1b7b947a7f66c02dcaa927f3203b87871c114ba10/pyarrow-22.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7388ac685cab5b279a41dfe0a6ccd99e4dbf322edfb63e02fc0443bf24134e91", size = 47703663, upload-time = "2025-10-24T10:09:27.369Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/62/45abedde480168e83a1de005b7b7043fd553321c1e8c5a9a114425f64842/pyarrow-22.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f633074f36dbc33d5c05b5dc75371e5660f1dbf9c8b1d95669def05e5425989c", size = 48066543, upload-time = "2025-10-24T10:09:34.908Z" },
+    { url = "https://files.pythonhosted.org/packages/84/e9/7878940a5b072e4f3bf998770acafeae13b267f9893af5f6d4ab3904b67e/pyarrow-22.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4c19236ae2402a8663a2c8f21f1870a03cc57f0bef7e4b6eb3238cc82944de80", size = 50288838, upload-time = "2025-10-24T10:09:44.394Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/03/f335d6c52b4a4761bcc83499789a1e2e16d9d201a58c327a9b5cc9a41bd9/pyarrow-22.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c34fe18094686194f204a3b1787a27456897d8a2d62caf84b61e8dfbc0252ae", size = 29185594, upload-time = "2025-10-24T10:09:53.111Z" },
+]
+
 [[package]]
 name = "pycparser"
 version = "2.22"
@@ -2766,6 +2983,35 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" },
 ]
 
+[[package]]
+name = "pygame"
+version = "2.6.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/cc/08bba60f00541f62aaa252ce0cfbd60aebd04616c0b9574f755b583e45ae/pygame-2.6.1.tar.gz", hash = "sha256:56fb02ead529cee00d415c3e007f75e0780c655909aaa8e8bf616ee09c9feb1f", size = 14808125, upload-time = "2024-09-29T13:41:34.698Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c4/ca/8f367cb9fe734c4f6f6400e045593beea2635cd736158f9fabf58ee14e3c/pygame-2.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:20349195326a5e82a16e351ed93465a7845a7e2a9af55b7bc1b2110ea3e344e1", size = 13113753, upload-time = "2024-09-29T14:26:13.751Z" },
+    { url = "https://files.pythonhosted.org/packages/83/47/6edf2f890139616b3219be9cfcc8f0cb8f42eb15efd59597927e390538cb/pygame-2.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f3935459109da4bb0b3901da9904f0a3e52028a3332a355d298b1673a334cf21", size = 12378146, upload-time = "2024-09-29T14:26:22.456Z" },
+    { url = "https://files.pythonhosted.org/packages/00/9e/0d8aa8cf93db2d2ee38ebaf1c7b61d0df36ded27eb726221719c150c673d/pygame-2.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c31dbdb5d0217f32764797d21c2752e258e5fb7e895326538d82b5f75a0cd856", size = 13611760, upload-time = "2024-09-29T11:10:47.317Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/9e/d06adaa5cc65876bcd7a24f59f67e07f7e4194e6298130024ed3fb22c456/pygame-2.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:173badf82fa198e6888017bea40f511cb28e69ecdd5a72b214e81e4dcd66c3b1", size = 14298054, upload-time = "2024-09-29T11:39:53.891Z" },
+    { url = "https://files.pythonhosted.org/packages/7a/a1/9ae2852ebd3a7cc7d9ae7ff7919ab983e4a5c1b7a14e840732f23b2b48f6/pygame-2.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce8cc108b92de9b149b344ad2e25eedbe773af0dc41dfb24d1f07f679b558c60", size = 13977107, upload-time = "2024-09-29T11:39:56.831Z" },
+    { url = "https://files.pythonhosted.org/packages/31/df/6788fd2e9a864d0496a77670e44a7c012184b7a5382866ab0e60c55c0f28/pygame-2.6.1-cp311-cp311-win32.whl", hash = "sha256:811e7b925146d8149d79193652cbb83e0eca0aae66476b1cb310f0f4226b8b5c", size = 10250863, upload-time = "2024-09-29T11:44:48.199Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/55/ca3eb851aeef4f6f2e98a360c201f0d00bd1ba2eb98e2c7850d80aabc526/pygame-2.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:91476902426facd4bb0dad4dc3b2573bc82c95c71b135e0daaea072ed528d299", size = 10622016, upload-time = "2024-09-29T12:17:01.545Z" },
+    { url = "https://files.pythonhosted.org/packages/92/16/2c602c332f45ff9526d61f6bd764db5096ff9035433e2172e2d2cadae8db/pygame-2.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4ee7f2771f588c966fa2fa8b829be26698c9b4836f82ede5e4edc1a68594942e", size = 13118279, upload-time = "2024-09-29T14:26:30.427Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/53/77ccbc384b251c6e34bfd2e734c638233922449a7844e3c7a11ef91cee39/pygame-2.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c8040ea2ab18c6b255af706ec01355c8a6b08dc48d77fd4ee783f8fc46a843bf", size = 12384524, upload-time = "2024-09-29T14:26:49.996Z" },
+    { url = "https://files.pythonhosted.org/packages/06/be/3ed337583f010696c3b3435e89a74fb29d0c74d0931e8f33c0a4246307a9/pygame-2.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47a6938de93fa610accd4969e638c2aebcb29b2fca518a84c3a39d91ab47116", size = 13587123, upload-time = "2024-09-29T11:10:50.072Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/ca/b015586a450db59313535662991b34d24c1f0c0dc149cc5f496573900f4e/pygame-2.6.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33006f784e1c7d7e466fcb61d5489da59cc5f7eb098712f792a225df1d4e229d", size = 14275532, upload-time = "2024-09-29T11:39:59.356Z" },
+    { url = "https://files.pythonhosted.org/packages/b9/f2/d31e6ad42d657af07be2ffd779190353f759a07b51232b9e1d724f2cda46/pygame-2.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1206125f14cae22c44565c9d333607f1d9f59487b1f1432945dfc809aeaa3e88", size = 13952653, upload-time = "2024-09-29T11:40:01.781Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/42/8ea2a6979e6fa971702fece1747e862e2256d4a8558fe0da6364dd946c53/pygame-2.6.1-cp312-cp312-win32.whl", hash = "sha256:84fc4054e25262140d09d39e094f6880d730199710829902f0d8ceae0213379e", size = 10252421, upload-time = "2024-09-29T11:14:26.877Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/90/7d766d54bb95939725e9a9361f9c06b0cfbe3fe100aa35400f0a461a278a/pygame-2.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:3a9e7396be0d9633831c3f8d5d82dd63ba373ad65599628294b7a4f8a5a01a65", size = 10624591, upload-time = "2024-09-29T11:52:54.489Z" },
+    { url = "https://files.pythonhosted.org/packages/e1/91/718acf3e2a9d08a6ddcc96bd02a6f63c99ee7ba14afeaff2a51c987df0b9/pygame-2.6.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ae6039f3a55d800db80e8010f387557b528d34d534435e0871326804df2a62f2", size = 13090765, upload-time = "2024-09-29T14:27:02.377Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/c6/9cb315de851a7682d9c7568a41ea042ee98d668cb8deadc1dafcab6116f0/pygame-2.6.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2a3a1288e2e9b1e5834e425bedd5ba01a3cd4902b5c2bff8ed4a740ccfe98171", size = 12381704, upload-time = "2024-09-29T14:27:10.228Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/8f/617a1196e31ae3b46be6949fbaa95b8c93ce15e0544266198c2266cc1b4d/pygame-2.6.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27eb17e3dc9640e4b4683074f1890e2e879827447770470c2aba9f125f74510b", size = 13581091, upload-time = "2024-09-29T11:30:27.653Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/87/2851a564e40a2dad353f1c6e143465d445dab18a95281f9ea458b94f3608/pygame-2.6.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c1623180e70a03c4a734deb9bac50fc9c82942ae84a3a220779062128e75f3b", size = 14273844, upload-time = "2024-09-29T11:40:04.138Z" },
+    { url = "https://files.pythonhosted.org/packages/85/b5/aa23aa2e70bcba42c989c02e7228273c30f3b44b9b264abb93eaeff43ad7/pygame-2.6.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef07c0103d79492c21fced9ad68c11c32efa6801ca1920ebfd0f15fb46c78b1c", size = 13951197, upload-time = "2024-09-29T11:40:06.785Z" },
+    { url = "https://files.pythonhosted.org/packages/a6/06/29e939b34d3f1354738c7d201c51c250ad7abefefaf6f8332d962ff67c4b/pygame-2.6.1-cp313-cp313-win32.whl", hash = "sha256:3acd8c009317190c2bfd81db681ecef47d5eb108c2151d09596d9c7ea9df5c0e", size = 10249309, upload-time = "2024-09-29T11:10:23.329Z" },
+    { url = "https://files.pythonhosted.org/packages/7e/11/17f7f319ca91824b86557e9303e3b7a71991ef17fd45286bf47d7f0a38e6/pygame-2.6.1-cp313-cp313-win_amd64.whl", hash = "sha256:813af4fba5d0b2cb8e58f5d95f7910295c34067dcc290d34f1be59c48bd1ea6a", size = 10620084, upload-time = "2024-09-29T11:48:51.587Z" },
+]
+
 [[package]]
 name = "pygments"
 version = "2.19.1"
@@ -2775,6 +3021,22 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload-time = "2025-01-06T17:26:25.553Z" },
 ]
 
+[[package]]
+name = "pynndescent"
+version = "0.6.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "joblib" },
+    { name = "llvmlite" },
+    { name = "numba" },
+    { name = "scikit-learn" },
+    { name = "scipy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4a/fb/7f58c397fb31666756457ee2ac4c0289ef2daad57f4ae4be8dec12f80b03/pynndescent-0.6.0.tar.gz", hash = "sha256:7ffde0fb5b400741e055a9f7d377e3702e02250616834231f6c209e39aac24f5", size = 2992987, upload-time = "2026-01-08T21:29:58.943Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b2/e6/94145d714402fd5ade00b5661f2d0ab981219e07f7db9bfa16786cdb9c04/pynndescent-0.6.0-py3-none-any.whl", hash = "sha256:dc8c74844e4c7f5cbd1e0cd6909da86fdc789e6ff4997336e344779c3d5538ef", size = 73511, upload-time = "2026-01-08T21:29:57.306Z" },
+]
+
 [[package]]
 name = "pyparsing"
 version = "3.2.3"
@@ -3469,34 +3731,81 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" },
 ]
 
+[[package]]
+name = "statsmodels"
+version = "0.14.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numpy" },
+    { name = "packaging" },
+    { name = "pandas" },
+    { name = "patsy" },
+    { name = "scipy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a", size = 20689085, upload-time = "2025-12-05T23:08:39.522Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a9/4d/df4dd089b406accfc3bb5ee53ba29bb3bdf5ae61643f86f8f604baa57656/statsmodels-0.14.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6ad5c2810fc6c684254a7792bf1cbaf1606cdee2a253f8bd259c43135d87cfb4", size = 10121514, upload-time = "2025-12-05T19:28:16.521Z" },
+    { url = "https://files.pythonhosted.org/packages/82/af/ec48daa7f861f993b91a0dcc791d66e1cf56510a235c5cbd2ab991a31d5c/statsmodels-0.14.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:341fa68a7403e10a95c7b6e41134b0da3a7b835ecff1eb266294408535a06eb6", size = 10003346, upload-time = "2025-12-05T19:28:29.568Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/2c/c8f7aa24cd729970728f3f98822fb45149adc216f445a9301e441f7ac760/statsmodels-0.14.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdf1dfe2a3ca56f5529118baf33a13efed2783c528f4a36409b46bbd2d9d48eb", size = 10129872, upload-time = "2025-12-05T23:09:25.724Z" },
+    { url = "https://files.pythonhosted.org/packages/40/c6/9ae8e9b0721e9b6eb5f340c3a0ce8cd7cce4f66e03dd81f80d60f111987f/statsmodels-0.14.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3764ba8195c9baf0925a96da0743ff218067a269f01d155ca3558deed2658ca", size = 10381964, upload-time = "2025-12-05T23:09:41.326Z" },
+    { url = "https://files.pythonhosted.org/packages/28/8c/cf3d30c8c2da78e2ad1f50ade8b7fabec3ff4cdfc56fbc02e097c4577f90/statsmodels-0.14.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9e8d2e519852adb1b420e018f5ac6e6684b2b877478adf7fda2cfdb58f5acb5d", size = 10409611, upload-time = "2025-12-05T23:09:57.131Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/cc/018f14ecb58c6cb89de9d52695740b7d1f5a982aa9ea312483ea3c3d5f77/statsmodels-0.14.6-cp311-cp311-win_amd64.whl", hash = "sha256:2738a00fca51196f5a7d44b06970ace6b8b30289839e4808d656f8a98e35faa7", size = 9580385, upload-time = "2025-12-05T19:28:42.778Z" },
+    { url = "https://files.pythonhosted.org/packages/25/ce/308e5e5da57515dd7cab3ec37ea2d5b8ff50bef1fcc8e6d31456f9fae08e/statsmodels-0.14.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe76140ae7adc5ff0e60a3f0d56f4fffef484efa803c3efebf2fcd734d72ecb5", size = 10091932, upload-time = "2025-12-05T19:28:55.446Z" },
+    { url = "https://files.pythonhosted.org/packages/05/30/affbabf3c27fb501ec7b5808230c619d4d1a4525c07301074eb4bda92fa9/statsmodels-0.14.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:26d4f0ed3b31f3c86f83a92f5c1f5cbe63fc992cd8915daf28ca49be14463a1c", size = 9997345, upload-time = "2025-12-05T19:29:10.278Z" },
+    { url = "https://files.pythonhosted.org/packages/48/f5/3a73b51e6450c31652c53a8e12e24eac64e3824be816c0c2316e7dbdcb7d/statsmodels-0.14.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8c00a42863e4f4733ac9d078bbfad816249c01451740e6f5053ecc7db6d6368", size = 10058649, upload-time = "2025-12-05T23:10:12.775Z" },
+    { url = "https://files.pythonhosted.org/packages/81/68/dddd76117df2ef14c943c6bbb6618be5c9401280046f4ddfc9fb4596a1b8/statsmodels-0.14.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b58cf7474aa9e7e3b0771a66537148b2df9b5884fbf156096c0e6c1ff0469d", size = 10339446, upload-time = "2025-12-05T23:10:28.503Z" },
+    { url = "https://files.pythonhosted.org/packages/56/4a/dce451c74c4050535fac1ec0c14b80706d8fc134c9da22db3c8a0ec62c33/statsmodels-0.14.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:81e7dcc5e9587f2567e52deaff5220b175bf2f648951549eae5fc9383b62bc37", size = 10368705, upload-time = "2025-12-05T23:10:44.339Z" },
+    { url = "https://files.pythonhosted.org/packages/60/15/3daba2df40be8b8a9a027d7f54c8dedf24f0d81b96e54b52293f5f7e3418/statsmodels-0.14.6-cp312-cp312-win_amd64.whl", hash = "sha256:b5eb07acd115aa6208b4058211138393a7e6c2cf12b6f213ede10f658f6a714f", size = 9543991, upload-time = "2025-12-05T23:10:58.536Z" },
+    { url = "https://files.pythonhosted.org/packages/81/59/a5aad5b0cc266f5be013db8cde563ac5d2a025e7efc0c328d83b50c72992/statsmodels-0.14.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:47ee7af083623d2091954fa71c7549b8443168f41b7c5dce66510274c50fd73e", size = 10072009, upload-time = "2025-12-05T23:11:14.021Z" },
+    { url = "https://files.pythonhosted.org/packages/53/dd/d8cfa7922fc6dc3c56fa6c59b348ea7de829a94cd73208c6f8202dd33f17/statsmodels-0.14.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa60d82e29fcd0a736e86feb63a11d2380322d77a9369a54be8b0965a3985f71", size = 9980018, upload-time = "2025-12-05T23:11:30.907Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/77/0ec96803eba444efd75dba32f2ef88765ae3e8f567d276805391ec2c98c6/statsmodels-0.14.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ee7d595f5939cc20bf946faedcb5137d975f03ae080f300ebb4398f16a5bd4", size = 10060269, upload-time = "2025-12-05T23:11:46.338Z" },
+    { url = "https://files.pythonhosted.org/packages/10/b9/fd41f1f6af13a1a1212a06bb377b17762feaa6d656947bf666f76300fc05/statsmodels-0.14.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:730f3297b26749b216a06e4327fe0be59b8d05f7d594fb6caff4287b69654589", size = 10324155, upload-time = "2025-12-05T23:12:01.805Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/0f/a6900e220abd2c69cd0a07e3ad26c71984be6061415a60e0f17b152ecf08/statsmodels-0.14.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f1c08befa85e93acc992b72a390ddb7bd876190f1360e61d10cf43833463bc9c", size = 10349765, upload-time = "2025-12-05T23:12:18.018Z" },
+    { url = "https://files.pythonhosted.org/packages/98/08/b79f0c614f38e566eebbdcff90c0bcacf3c6ba7a5bbb12183c09c29ca400/statsmodels-0.14.6-cp313-cp313-win_amd64.whl", hash = "sha256:8021271a79f35b842c02a1794465a651a9d06ec2080f76ebc3b7adce77d08233", size = 9540043, upload-time = "2025-12-05T23:12:33.887Z" },
+    { url = "https://files.pythonhosted.org/packages/71/de/09540e870318e0c7b58316561d417be45eff731263b4234fdd2eee3511a8/statsmodels-0.14.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:00781869991f8f02ad3610da6627fd26ebe262210287beb59761982a8fa88cae", size = 10069403, upload-time = "2025-12-05T23:12:48.424Z" },
+    { url = "https://files.pythonhosted.org/packages/ab/f0/63c1bfda75dc53cee858006e1f46bd6d6f883853bea1b97949d0087766ca/statsmodels-0.14.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:73f305fbf31607b35ce919fae636ab8b80d175328ed38fdc6f354e813b86ee37", size = 9989253, upload-time = "2025-12-05T23:13:05.274Z" },
+    { url = "https://files.pythonhosted.org/packages/c1/98/b0dfb4f542b2033a3341aa5f1bdd97024230a4ad3670c5b0839d54e3dcab/statsmodels-0.14.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e443e7077a6e2d3faeea72f5a92c9f12c63722686eb80bb40a0f04e4a7e267ad", size = 10090802, upload-time = "2025-12-05T23:13:20.653Z" },
+    { url = "https://files.pythonhosted.org/packages/34/0e/2408735aca9e764643196212f9069912100151414dd617d39ffc72d77eee/statsmodels-0.14.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3414e40c073d725007a6603a18247ab7af3467e1af4a5e5a24e4c27bc26673b4", size = 10337587, upload-time = "2025-12-05T23:13:37.597Z" },
+    { url = "https://files.pythonhosted.org/packages/0f/36/4d44f7035ab3c0b2b6a4c4ebb98dedf36246ccbc1b3e2f51ebcd7ac83abb/statsmodels-0.14.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a518d3f9889ef920116f9fa56d0338069e110f823926356946dae83bc9e33e19", size = 10363350, upload-time = "2025-12-05T23:13:53.08Z" },
+    { url = "https://files.pythonhosted.org/packages/26/33/f1652d0c59fa51de18492ee2345b65372550501ad061daa38f950be390b6/statsmodels-0.14.6-cp314-cp314-win_amd64.whl", hash = "sha256:151b73e29f01fe619dbce7f66d61a356e9d1fe5e906529b78807df9189c37721", size = 9588010, upload-time = "2025-12-05T23:14:07.28Z" },
+]
+
 [[package]]
 name = "studies"
 version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
+    { name = "accelerate" },
     { name = "catboost" },
+    { name = "datasets" },
     { name = "faiss-cpu" },
+    { name = "gymnasium", extra = ["toy-text"] },
     { name = "imblearn" },
     { name = "ipykernel" },
     { name = "langchain" },
     { name = "langchain-community" },
     { name = "langchain-huggingface" },
     { name = "langchain-ollama" },
+    { name = "langchain-text-splitters" },
     { name = "matplotlib" },
     { name = "nbformat" },
     { name = "numpy" },
     { name = "opencv-python" },
+    { name = "openpyxl" },
     { name = "pandas" },
     { name = "pandas-stubs" },
     { name = "plotly" },
+    { name = "polars" },
     { name = "pypdf" },
     { name = "scikit-learn" },
     { name = "scipy" },
     { name = "seaborn" },
     { name = "sentence-transformers" },
     { name = "shap" },
+    { name = "statsmodels" },
     { name = "tensorflow" },
     { name = "tf-keras" },
+    { name = "umap-learn" },
     { name = "xgboost" },
     { name = "yfinance" },
 ]
@@ -3509,29 +3818,37 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "accelerate", specifier = ">=1.12.0" },
     { name = "catboost", specifier = ">=1.2.8" },
+    { name = "datasets", specifier = ">=4.4.2" },
     { name = "faiss-cpu", specifier = ">=1.13.2" },
+    { name = "gymnasium", extras = ["toy-text"], specifier = ">=1.2.3" },
     { name = "imblearn", specifier = ">=0.0" },
     { name = "ipykernel", specifier = ">=6.29.5" },
     { name = "langchain", specifier = ">=1.2.0" },
     { name = "langchain-community", specifier = ">=0.4.1" },
     { name = "langchain-huggingface", specifier = ">=1.2.0" },
     { name = "langchain-ollama", specifier = ">=1.0.1" },
+    { name = "langchain-text-splitters", specifier = ">=1.1.0" },
     { name = "matplotlib", specifier = ">=3.10.1" },
     { name = "nbformat", specifier = ">=5.10.4" },
     { name = "numpy", specifier = ">=2.2.5" },
     { name = "opencv-python", specifier = ">=4.11.0.86" },
+    { name = "openpyxl", specifier = ">=3.1.5" },
     { name = "pandas", specifier = ">=2.2.3" },
     { name = "pandas-stubs", specifier = ">=2.3.2.250926" },
     { name = "plotly", specifier = ">=6.3.0" },
+    { name = "polars", specifier = ">=1.37.0" },
     { name = "pypdf", specifier = ">=6.5.0" },
     { name = "scikit-learn", specifier = ">=1.6.1" },
     { name = "scipy", specifier = ">=1.15.2" },
     { name = "seaborn", specifier = ">=0.13.2" },
     { name = "sentence-transformers", specifier = ">=5.2.0" },
     { name = "shap", specifier = ">=0.49.1" },
+    { name = "statsmodels", specifier = ">=0.14.6" },
     { name = "tensorflow", specifier = ">=2.20.0" },
     { name = "tf-keras", specifier = ">=2.20.1" },
+    { name = "umap-learn", specifier = ">=0.5.11" },
     { name = "xgboost", specifier = ">=3.1.2" },
     { name = "yfinance", specifier = ">=0.2.66" },
 ]
@@ -3872,6 +4189,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
 ]
 
+[[package]]
+name = "umap-learn"
+version = "0.5.11"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "numba" },
+    { name = "numpy" },
+    { name = "pynndescent" },
+    { name = "scikit-learn" },
+    { name = "scipy" },
+    { name = "tqdm" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/94/9a/a1e4a257a9aa979dac4f6d5781dac929cbb0949959e2003ed82657d10b0f/umap_learn-0.5.11.tar.gz", hash = "sha256:31566ffd495fbf05d7ab3efcba703861c0f5e6fc6998a838d0e2becdd00e54f5", size = 96409, upload-time = "2026-01-12T20:44:47.553Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/d2/fcf7192dd1cd8c090b6cfd53fa223c4fb2887a17c47e06bc356d44f40dfb/umap_learn-0.5.11-py3-none-any.whl", hash = "sha256:cb17adbde9d544ba79481b3ab4d81ac222e940f3d9219307bea6044f869af3cc", size = 90890, upload-time = "2026-01-12T20:44:46.511Z" },
+]
+
 [[package]]
 name = "urllib3"
 version = "2.5.0"