競馬予想AIを改良する1~親馬の成績~

1class PedigreeResults:
2    columns = [
3        "日付",
4        "開催",
5        "天 気",
6        "R",
7        "レース名",
8        "映 像",
9        "頭 数",
10        "枠 番",
11        "馬 番",
12        "オ ッ ズ",
13        "人 気",
14        "着 順",
15        "騎手",
16        "斤 量",
17        "距離",
18        "馬 場",
19        "馬場 指数",
20        "タイム",
21        "着差",
22        "ﾀｲﾑ 指数",
23        "通過",
24        "ペース",
25        "上り",
26        "馬体重",
27        "厩舎 ｺﾒﾝﾄ",
28        "備考",
29        "勝ち馬 (2着馬)",
30        "賞金",
31    ]
32    headers = {
33        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36"
34    }
35
36    def __init__(self, horse_path: str, save_path: str) -> None:
37        with open(horse_path, "rb") as f:
38            self.horse_ids = pickle.load(f)
39        self.save_path = save_path
40
41    def df_process_sire_results(self, df: pd.DataFrame) -> pd.DataFrame:
42        try:
43            df.columns = ["_".join(col).strip() for col in df.columns.values]
44            df = df[df["年度_年度"] != "累計"]
45        except:
46            df = None
47        return df
48
49    def scraping_result(self, horse_id: str) -> pd.DataFrame | None:
50        """親馬の過去成績を取得する
51
52        Args:
53            horse_id (str): 親馬のID
54
55        Raises:
56            e: 親馬の過去成績の取得ができなかった場合はエラーをそのまま返す
57
58        Returns:
59            pd.DataFrame | None: 親馬の過去成績
60        """
61        isNotExist = True
62        url = f"https://db.netkeiba.com/horse/{horse_id}"
63        response = requests.get(url, headers=self.headers)
64        response.encoding = "EUC-JP"
65        html_string = io.StringIO(response.text)
66        for i in range(1, 5):
67            try:
68                df = pd.read_html(html_string)[i]
69                if df.columns[0] != "日付":
70                    continue
71                else:
72                    isNotExist = False
73                    break
74            except IndexError as e:
75                df = pd.DataFrame(0, index=range(1), columns=self.columns)
76            except Exception as e:
77                print(horse_id)
78                raise e
79        if isNotExist:
80            df = pd.DataFrame(0, index=range(1), columns=self.columns)
81        return df
82
83    def scraping_sire_results(self, horse_id: str) -> pd.DataFrame | None:
84        try:
85            url = f"https://db.netkeiba.com/?pid=horse_sire&id={horse_id}&course=1&mode=1&type=0"
86            response = requests.get(url, headers=self.headers)
87
88            response.encoding = "EUC-JP"
89            soup = bs(response.text, "html.parser")
90            target_element1 = soup.select_one('table[summary="産駒成績"]')
91            target_element2 = soup.select_one('table[summary="成績"]')
92            df_tmp1 = (
93                pd.read_html(io.StringIO(str(target_element1)))[0]
94                if target_element1 != None
95                else None
96            )
97            df_tmp2 = (
98                pd.read_html(io.StringIO(str(target_element2)))[0]
99                if target_element2 != None
100                else None
101            )
102            df = {
103                "sire_results": self.df_process_sire_results(df_tmp1),
104                "sire_results_BMS": self.df_process_sire_results(df_tmp2),
105            }
106        except Exception as e:
107            print(horse_id)
108            raise e
109        return df
110
111    def scraping_sire_course(self, horse_id: str) -> pd.DataFrame | None:
112        try:
113            url = f"https://db.netkeiba.com/?pid=horse_sire&id={horse_id}&course=1&mode=1&type=1"
114            response = requests.get(url, headers=self.headers)
115            response.encoding = "EUC-JP"
116            html_string = io.StringIO(response.text)
117            try:
118                tables = pd.read_html(html_string)
119            except:
120                return [None, None]
121            dfs = []
122            first_loop = True
123            for i in tables:
124                tmp = i.copy()
125                if tmp.iloc[0, 0] in ["輸入年", "供用開始年"]:
126                    continue
127                # マルチインデックスを1つにする
128                tmp.columns = ["_".join(col).strip() for col in tmp.columns.values]
129                # 累計の行は削除
130                tmp = tmp[tmp["年度_年度"] != "累計"]
131                # 初回ループはデータを格納する
132                if first_loop:
133                    first_loop = False
134                    dfs.append(tmp)
135                    continue
136                # 次のブロックに移る
137                elif tmp.columns[1] == "芝・良_1着":
138                    dfs.append(tmp)
139                    continue
140                else:
141                    dfs[-1] = pd.merge(dfs[-1], tmp, on="年度_年度", how="inner")
142            # BeautifulSoupでパース
143            soup = bs(response.text, "html.parser")
144            # テーブルを含む要素がなければNoneを入れる
145            target_element1 = soup.select_one('table[summary="産駒成績"]')
146            target_element2 = soup.select_one('table[summary="成績"]')
147            if target_element1 == None:
148                dfs.insert(0, None)
149            if target_element2 == None:
150                dfs.append(None)
151        except Exception as e:
152            print(horse_id)
153            raise e
154        return dfs
155
156    def scraping_sire_distance(self, horse_id: str) -> pd.DataFrame | None:
157        try:
158            url = f"https://db.netkeiba.com/?pid=horse_sire&id={horse_id}&course=1&mode=1&type=2"
159            response = requests.get(url, headers=self.headers)
160            response.encoding = "EUC-JP"
161            html_string = io.StringIO(response.text)
162            try:
163                tables = pd.read_html(html_string)
164            except:
165                return [None, None]
166            dfs = []
167            first_loop = True
168            for i in tables:
169                tmp = i.copy()
170                if tmp.iloc[0, 0] in ["輸入年", "供用開始年"]:
171                    continue
172                # マルチインデックスを1つにする
173                tmp.columns = ["_".join(col).strip() for col in tmp.columns.values]
174                # 累計の行は削除
175                tmp = tmp[tmp["年度_年度"] != "累計"]
176                # 全データ型をintに変換
177                tmp = tmp.astype(int)
178                # 初回ループはデータを格納する
179                if first_loop:
180                    first_loop = False
181                    dfs.append(tmp)
182                    continue
183                # 次のブロックに移る
184                elif tmp.columns[1] == "-1400(芝)_1着":
185                    dfs.append(tmp)
186                    continue
187                else:
188                    dfs[-1] = pd.merge(dfs[-1], tmp, on="年度_年度", how="inner")
189            # BeautifulSoupでパース
190            soup = bs(response.text, "html.parser")
191            # テーブルを含む要素がなければNoneを入れる
192            target_element1 = soup.select_one('table[summary="産駒成績"]')
193            target_element2 = soup.select_one('table[summary="成績"]')
194            if target_element1 == None:
195                dfs.insert(0, None)
196            if target_element2 == None:
197                dfs.append(None)
198        except Exception as e:
199            print(horse_id)
200            raise e
201        return dfs
202
203    def save(self, data: dict | None, name: str) -> None:
204        """レース結果を保存する
205
206        Args:
207            data (dict): レース結果の辞書
208            name (str): 保存するファイル名
209        """
210        if data is None:
211            return
212        with open(f"{self.save_path}/{name}.pickle", "wb") as f:
213            pickle.dump(data, f)
214
215    def scrape_save(self, sleep_rate=2) -> None:
216        tim_add = 0
217        for horse_id in self.horse_ids:
218            # 既にファイルが作成されていればスルー
219            if os.path.isfile(f"{self.save_path}/{horse_id}.pickle"):
220                continue
221            start = time.perf_counter()
222            df_result = self.scraping_result(horse_id)
223            end = time.perf_counter()
224            time.sleep(sleep_rate - (end - start))
225
226            start = time.perf_counter()
227            df_sire_results = self.scraping_sire_results(horse_id)
228            end = time.perf_counter()
229            time.sleep(sleep_rate - (end - start))
230
231            start = time.perf_counter()
232            df_sire_course = self.scraping_sire_course(horse_id)
233            end = time.perf_counter()
234            time.sleep(sleep_rate - (end - start))
235
236            start = time.perf_counter()
237            df_sire_distance = self.scraping_sire_distance(horse_id)
238            end = time.perf_counter()
239            time.sleep(sleep_rate - (end - start))
240
241            df = {
242                "result": df_result,
243                "sire_results": df_sire_results,
244                "sire_course": df_sire_course,
245                "sire_distance": df_sire_distance,
246            }
247            self.save(df, horse_id)

1class PedigreeSireProcessor:
2    def __init__(self, path) -> None:
3        df = pd.read_pickle(path)
4        self.sire_results_raw = df["sire_results"]
5        self.sire_course_raw = df["sire_course"]
6        self.sire_distance_raw = df["sire_distance"]
7        # 整形
8        try:
9            self.sire_results = self.adjust(self.sire_results_raw["sire_results"])
10        except:
11            self.sire_results = pd.read_pickle("../template/sire_results.pkl")
12        try:
13            self.sire_results_bms = self.adjust(
14                self.sire_results_raw["sire_results_BMS"]
15            )
16        except:
17            self.sire_results_bms = pd.read_pickle("../template/sire_results_BMS.pkl")
18        try:
19            self.sire_course = self.adjust(self.sire_course_raw[0])
20        except:
21            self.sire_course = pd.read_pickle("../template/sire_course.pkl")
22        try:
23            self.sire_course_bms = self.adjust(self.sire_course_raw[1])
24        except:
25            self.sire_course_bms = pd.read_pickle("../template/sire_course_BMS.pkl")
26        try:
27            self.sire_distance = self.adjust(self.sire_distance_raw[0])
28        except:
29            self.sire_distance = pd.read_pickle("../template/sire_distance.pkl")
30        try:
31            self.sire_distance_bms = self.adjust(self.sire_distance_raw[1])
32        except:
33            self.sire_distance_bms = pd.read_pickle("../template/sire_distance_BMS.pkl")
34
35    def adjust(self, df: pd.DataFrame) -> pd.DataFrame:
36        """カラム名を変更する
37
38        Args:
39            df (pd.DataFrame): 変換前データ
40
41        Returns:
42            pd.DataFrame: 変換後データ
43        """
44        df.rename(columns=lambda x: re.sub(r"\s", "", x), inplace=True)
45        df.rename(columns=lambda x: re.sub(r"_", "", x), inplace=True)
46        df.rename(columns=lambda x: re.sub(r"^(.*)\1$", r"\1", x), inplace=True)
47        return df
48
49    def translate_results(self) -> pd.DataFrame:
50        """競争成績別データの整形をする
51
52        Returns:
53            pd.DataFrame: 変換後データ
54        """
55        dfs = []
56        for i, df in enumerate([self.sire_results, self.sire_results_bms]):
57            df = df.copy()
58            df.drop(columns=["年度", "順位", "勝馬率", "代表馬"], axis=1, inplace=True)
59            df2 = pd.DataFrame(df.mean()).T
60            df2["len"] = len(df)
61            df2 = df2.add_prefix(f"{i+1}_")
62            dfs.append(df2)
63        return pd.concat(dfs, axis=1)
64
65    def translate_course(self) -> pd.DataFrame:
66        """コース・馬場別データの整形をする
67
68        Returns:
69            pd.DataFrame: 変換後データ
70        """
71        dfs = []
72        for i, df in enumerate([self.sire_course, self.sire_course_bms]):
73            df = df.copy()
74            df.drop(columns=["年度"], axis=1, inplace=True)
75            df2 = pd.DataFrame(df.mean()).T
76            df2["len"] = len(df)
77            df2 = df2.add_prefix(f"{i+1}_")
78            dfs.append(df2)
79        return pd.concat(dfs, axis=1)
80
81    def translate_distance(self) -> pd.DataFrame:
82        """距離別データの整形をする
83
84        Returns:
85            pd.DataFrame: 変換後データ
86        """
87        dfs = []
88        for i, df in enumerate([self.sire_distance, self.sire_distance_bms]):
89            df = df.copy()
90            df.drop(columns=["年度"], axis=1, inplace=True)
91            df2 = pd.DataFrame(df.mean()).T
92            df2["len"] = len(df)
93            df2 = df2.add_prefix(f"{i+1}_")
94            dfs.append(df2)
95        return pd.concat(dfs, axis=1)
96
97    def translate(self) -> pd.DataFrame:
98        """成績データの整形をする
99
100        Returns:
101            pd.DataFrame: _description_
102        """
103        dfs = [
104            self.translate_results(),
105            self.translate_course(),
106            self.translate_distance(),
107        ]
108        return pd.concat(dfs, axis=1)

1dir_list = os.listdir("../Raw-Data/Pedigree/")
2dfs = []
3for data in tqdm(dir_list):
4    save_path = f"../Processed-Data/Pedigree-Results/{data}"
5    peds = pd.read_pickle(f"../Raw-Data/Pedigree/{data}")
6    for i, ped in enumerate(peds):
7        try:
8            path = f"../Raw-Data/Pedigree-Results/{ped}.pickle"
9            sire = PedigreeSireProcessor(path)
10            df_sire = sire.translate()
11            dfs.append(df_sire)
12        except Exception as e:
13            print(path)
14            raise e
15df_all = pd.concat(dfs)
16
17# 標準化モデルの作成と適用
18scaler = StandardScaler()
19df_all_normalized = pd.DataFrame(
20    scaler.fit_transform(df_all), columns=df_all.columns, index=df_all.index
21)
22
23# 標準化モデルの保存
24with open("../models/v2/pedigree_results_scaler.pickle", "wb") as f:
25    pickle.dump(scaler, f)
26# 標準化されたデータの確認
27print("標準化後のデータ形状:", df_all_normalized.shape)
28print("\n標準化後の統計量:")
29print(df_all_normalized.describe())

1with open("../models/v2/pedigree_results_scaler.pickle", "rb") as f:
2    scaler = pickle.load(f)
3dir_list = os.listdir("../Raw-Data/Pedigree/")
4
5for data in tqdm(dir_list):
6    save_path = f"../Processed-Data/Pedigree-Results/{data}"
7    peds = pd.read_pickle(f"../Raw-Data/Pedigree/{data}")
8    if os.path.exists(save_path):
9        continue
10    for i, ped in enumerate(peds):
11        try:
12            path = f"../Raw-Data/Pedigree-Results/{ped}.pickle"
13            sire = PedigreeSireProcessor(path)
14            df_sire = sire.translate()
15
16        except Exception as e:
17            print(path)
18            raise e
19
20    # データの標準化
21    df_sire_normalized = pd.DataFrame(
22        scaler.transform(df_sire), columns=df_sire.columns, index=df_sire.index
23    )
24    df_sire_normalized.to_pickle(save_path)

1class AutoEncoder(nn.Module):
2    def __init__(
3        self,
4        input_dim: int,
5        hidden_dim: int,
6        latent_dim: int,
7        num_layers: int,
8        dropout: float,
9    ):
10        """
11        自己符号化器の初期化
12
13        Args:
14            input_dim (int): 入力データの次元数
15            hidden_dim (int): 各中間層のニューロン数
16            latent_dim (int): 圧縮後の次元数(潜在空間の次元数)
17            num_layers (int): 中間層の数
18            dropout (float): ドロップアウト率
19        """
20        super().__init__()
21
22        # エンコーダー部分の構築
23        encoder_layers = []
24        prev_dim = input_dim
25        for _ in range(num_layers):
26            encoder_layers.extend(
27                [
28                    nn.Linear(prev_dim, hidden_dim),
29                    # nn.LayerNorm(hidden_dim),
30                    nn.Mish(),
31                    nn.Dropout(dropout),
32                ]
33            )
34            prev_dim = hidden_dim
35        encoder_layers.append(nn.Linear(prev_dim, latent_dim))
36        self.encoder = nn.Sequential(*encoder_layers)
37
38        # デコーダー部分の構築
39        decoder_layers = []
40        prev_dim = latent_dim
41        for _ in range(num_layers):
42            decoder_layers.extend(
43                [
44                    nn.Linear(prev_dim, hidden_dim),
45                    # nn.LayerNorm(hidden_dim),
46                    nn.Mish(),
47                    nn.Dropout(dropout),
48                ]
49            )
50            prev_dim = hidden_dim
51        decoder_layers.append(nn.Linear(prev_dim, input_dim))
52        self.decoder = nn.Sequential(*decoder_layers)
53
54    def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
55        """
56        順伝播
57        Args:
58            x (torch.Tensor): 入力データ
59        Returns:
60            tuple[torch.Tensor, torch.Tensor]: (圧縮後のデータ, 再構成後のデータ)
61        """
62        encoded = self.encoder(x)  # エンコード
63        decoded = self.decoder(encoded)  # デコード
64        return encoded, decoded
65
66    def encode(self, x: torch.Tensor) -> torch.Tensor:
67        """
68        エンコードのみを行う
69        Args:
70            x (torch.Tensor): 入力データ
71        Returns:
72            torch.Tensor: 圧縮後のデータ
73        """
74        return self.encoder(x)
75
76    def decode(self, z: torch.Tensor) -> torch.Tensor:
77        """
78        デコードのみを行う
79        Args:
80            z (torch.Tensor): 圧縮されたデータ
81        Returns:
82            torch.Tensor: 再構成後のデータ
83        """
84        return self.decoder(z)

1def ae_loss(recon_x, x):
2    """AutoEncoderの損失関数
3    Args:
4        recon_x: 再構成されたデータ
5        x: 入力データ
6    Returns:
7        reconstruction_loss: 再構成誤差
8    """
9    reconstruction_loss = F.mse_loss(recon_x, x, reduction="sum")
10    return reconstruction_loss

1class PedigreeDataset(Dataset):
2    def __init__(self, file_paths):
3        self.file_paths = file_paths
4
5    def __len__(self):
6        return len(self.file_paths)
7
8    def __getitem__(self, idx):
9        df = pd.read_pickle(self.file_paths[idx])
10        df = df.astype("float32")
11        data_tensor = torch.tensor(df.values, dtype=torch.float32)
12        return data_tensor

1def objective(trial):
2    # ハイパーパラメータの設定
3    lr = trial.suggest_float("lr", 1e-5, 1e-2, log=True)
4    hidden_dim = trial.suggest_int("hidden_dim", 10, 200)
5    latent_dim = trial.suggest_int("latent_dim", 2, 180, log=True)
6    num_layers = trial.suggest_int("num_layers", 2, 10)
7    dropout = trial.suggest_float("dropout", 0.0, 0.5)
8
9    # モデルとオプティマイザの設定
10    model = AutoEncoder(
11        input_dim=226,
12        hidden_dim=hidden_dim,
13        latent_dim=latent_dim,
14        num_layers=num_layers,
15        dropout=dropout,
16    )
17    model.to(device)
18    optimizer = optim.AdamW(model.parameters(), lr=lr)
19
20    # 訓練ループ
21    for epoch in range(10):  # エポック数は適宜調整
22        model.train()
23        for batch in train_loader:
24            input_data = batch.to(device)
25
26            optimizer.zero_grad()
27            _, recon_batch = model(input_data)
28            loss = ae_loss(recon_batch, input_data)
29            loss.backward()
30            optimizer.step()
31
32    # 検証データセットでの性能評価
33    return loss.item()

1file_names = os.listdir("../Processed-Data/Pedigree-Results")
2file_paths = list(map(lambda x: "../Processed-Data/Pedigree-Results/" + x, file_names))
3
4# データの分割
5train_paths, test_paths = train_test_split(file_paths, test_size=0.3)
6train_paths, val_paths = train_test_split(train_paths, test_size=0.2)
7
8# データセットの作成
9train_dataset = PedigreeDataset(train_paths)
10val_dataset = PedigreeDataset(val_paths)
11test_dataset = PedigreeDataset(test_paths)
12
13# データローダーの設定
14batch_size = 1024
15train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
16val_loader = DataLoader(val_dataset, batch_size=batch_size)
17test_loader = DataLoader(test_dataset, batch_size=batch_size)
18
19device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
20
21model = AutoEncoder(
22    input_dim=226,
23    hidden_dim=147,
24    latent_dim=30,
25    num_layers=2,
26    dropout=0.2088016451903072,
27)
28model.to(device)
29optimizer = optim.AdamW(model.parameters(), lr=0.009785938947748174)
30
31
32# エポック数
33num_epochs = 200
34
35# 評価を行うエポック数
36eval_interval = 5
37
38# 誤差を記録するリスト
39train_losses = []
40val_losses = []
41
42# Early Stopping用のパラメータ
43best_val_loss = float("inf")
44patience_counter = 0
45min_improvement = 0.005  # n%の改善を期待
46patience = 5  # n回連続で改善が見られない場合に停止
47
48# 訓練ループ
49for epoch in range(num_epochs):
50    model.train()
51    train_loss = 0
52    for batch in train_loader:
53        input_data = batch.to(device)
54
55        optimizer.zero_grad()
56        _, recon_batch = model(input_data)
57        loss = ae_loss(recon_batch, input_data)
58        loss.backward()
59        optimizer.step()
60        train_loss += loss.item()
61
62    # エポックごとの平均誤差を計算
63    avg_train_loss = train_loss / len(train_loader.dataset)
64    train_losses.append(avg_train_loss)
65
66    # 検証データセットでモデルを評価
67    model.eval()
68    val_loss = 0
69    with torch.no_grad():
70        for val_batch in val_loader:
71            val_input = val_batch.to(device)
72            _, recon_batch = model(val_input)
73            loss = ae_loss(recon_batch, val_input)
74            val_loss += loss.item()
75
76    avg_val_loss = val_loss / len(val_loader.dataset)
77    val_losses.append(avg_val_loss)
78
79    # Early Stopping判定
80    if avg_val_loss < best_val_loss * (1 - min_improvement):
81        best_val_loss = avg_val_loss
82        patience_counter = 0
83    else:
84        patience_counter += 1
85
86    if epoch % eval_interval == 0 or epoch == num_epochs - 1:
87        print(f"Epoch {epoch}, Train Loss: {avg_train_loss}, Val Loss: {avg_val_loss}")
88
89    if patience_counter >= patience:
90        print(f"Early stopping triggered at epoch {epoch}")
91        break
92
93# 実際に実行されたエポック数
94final_epoch = len(train_losses)
95
96# 学習曲線をプロット
97plt.figure(figsize=(10, 6))
98plt.plot(range(final_epoch), train_losses, label="Training Loss")
99plt.plot(range(final_epoch), val_losses, label="Validation Loss")
100plt.xlabel("Epoch")
101plt.ylabel("Loss")
102plt.title("Training and Validation Loss")
103plt.legend()
104plt.grid(True)
105plt.show()

1model.eval()
2test_loss = 0
3with torch.no_grad():
4    for test_batch in test_loader:
5        test_input = test_batch.to(device)
6        _, recon_batch = model(test_input)
7        loss = ae_loss(recon_batch, test_input)
8        test_loss += loss.item()
9
10test_loss /= len(test_loader.dataset)
11print(f"Test Loss: {test_loss}")
12# Test Loss: 10.111261991165115
13
14torch.save(model.state_dict(), "../models/v2/pedigree_result_AE.pth")