When I join a competition, I research winning solutions from past similar competitions. It takes a lot of time to read and digest them, but it's an incredible source of ideas and knowledge. But what if we could learn from all the competitions? We've been given a list of Kaggle writeups in this competition, but there are so many of them! If only we could find a way to extract some structured data and analyze it... Well, it turns out that large language models (LLMs) [1] can help us extract structured data from unstructured writeups.

# 1. Generate a binary classification dataset
X, y = make_classification(n_samples=10000, n_features=10,flip_y=0.1, random_state=42)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 2. Set the smoothing factor
for alpha in np.linspace(0,0.1,20):
    y_train_smoothed = np.where(y_train == 1, 1 - alpha / 2, alpha / 2)

    model = Sequential([
        Dense(64, activation='relu', input_shape=(X.shape[1],)),
        Dense(1, activation='sigmoid')  # Sigmoid output for binary classification
    ])

    # 3. Compile the model
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

    # 4. Train the model with smoothed labels
    model.fit(
        X_train, y_train_smoothed,
        epochs=10,
        batch_size=32,
        validation_split=0.1,
        verbose=0  # Set verbose to 0 if you want silent training
    )

    # 5. Evaluate the model
    preds = model.predict(X_test,verbose=0).flatten()
    metric=brier_score_loss(y_true=y_test,y_proba=preds)

    print(f"alpha={alpha:.4f}, brier_score={metric:.4f}")

alpha=0.0000, brier_score=0.0990
alpha=0.0053, brier_score=0.0989
alpha=0.0105, brier_score=0.0984
alpha=0.0158, brier_score=0.0984
alpha=0.0211, brier_score=0.0999
alpha=0.0263, brier_score=0.0999
alpha=0.0316, brier_score=0.0995
alpha=0.0368, brier_score=0.0998
alpha=0.0421, brier_score=0.1006
alpha=0.0474, brier_score=0.1021
alpha=0.0526, brier_score=0.1018
alpha=0.0579, brier_score=0.1017
alpha=0.0632, brier_score=0.1014
alpha=0.0684, brier_score=0.1021
alpha=0.0737, brier_score=0.1021
alpha=0.0789, brier_score=0.1011
alpha=0.0842, brier_score=0.1045
alpha=0.0895, brier_score=0.1017
alpha=0.0947, brier_score=0.1027
alpha=0.1000, brier_score=0.1034

model.compile(optimizer='adam', loss=partial(tf.keras.losses.binary_crossentropy,
                  from_logits=False,
                  label_smoothing=alpha), metrics=['accuracy'])