arthur/handson-ml3
1
0
Fork 0
mirror of https://github.com/ArthurDanjou/handson-ml3.git synced 2026-01-14 12:14:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Update notebook to latest Colab library versions

Browse Source
This commit is contained in:
Aurélien Geron
2024-10-05 18:34:42 +13:00
parent 4d9eb70586
commit 0d6496cf79
1 changed files with 73 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files

546
12_custom_models_and_training_with_tensorflow.ipynb
View File

@@ -1850,7 +1850,7 @@
}
],
"source": [
"model.save(\"my_model_with_a_custom_loss\") # extra code – saving works fine"
"model.save(\"my_model_with_a_custom_loss.keras\") # extra code – saving works fine"
]
},
{
@@ -1859,7 +1859,7 @@
"metadata": {},
"outputs": [],
"source": [
"model = tf.keras.models.load_model(\"my_model_with_a_custom_loss\",\n",
"model = tf.keras.models.load_model(\"my_model_with_a_custom_loss.keras\",\n",
" custom_objects={\"huber_fn\": huber_fn})"
]
},
@@ -1982,7 +1982,7 @@
"metadata": {},
"outputs": [],
"source": [
"model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2\",\n",
"model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_threshold_2.keras\",\n",
" custom_objects={\"huber_fn\": create_huber(2.0)})"
]
},
@@ -2116,7 +2116,7 @@
}
],
"source": [
"model.save(\"my_model_with_a_custom_loss_class\") # extra code – saving works"
"model.save(\"my_model_with_a_custom_loss_class.keras\") # extra code – saving works"
]
},
{
@@ -2125,7 +2125,7 @@
"metadata": {},
"outputs": [],
"source": [
"model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_class\",\n",
"model = tf.keras.models.load_model(\"my_model_with_a_custom_loss_class.keras\",\n",
" custom_objects={\"HuberLoss\": HuberLoss})"
]
},
@@ -2220,80 +2220,6 @@
" kernel_constraint=my_positive_weights)"
]
},
{
"cell_type": "code",
"execution_count": 94,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/2\n",
"363/363 [==============================] - 1s 1ms/step - loss: 1.4714 - mae: 0.8316 - val_loss: inf - val_mae: inf\n",
"Epoch 2/2\n",
"363/363 [==============================] - 0s 1ms/step - loss: 0.8094 - mae: 0.6172 - val_loss: 2.6153 - val_mae: 0.6058\n",
"INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/2\n",
"363/363 [==============================] - 1s 1ms/step - loss: 0.6333 - mae: 0.5617 - val_loss: 1.1687 - val_mae: 0.5468\n",
"Epoch 2/2\n",
"363/363 [==============================] - 0s 1ms/step - loss: 0.5570 - mae: 0.5303 - val_loss: 1.0440 - val_mae: 0.5250\n"
]
},
{
"data": {
"text/plain": [
"<keras.src.callbacks.History at 0x19b868640>"
]
},
"execution_count": 94,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# extra code – show that building, training, saving, loading, and training again\n",
"# works fine with a model containing many custom parts\n",
"\n",
"tf.keras.utils.set_random_seed(42)\n",
"model = tf.keras.Sequential([\n",
" tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
" input_shape=input_shape),\n",
" tf.keras.layers.Dense(1, activation=my_softplus,\n",
" kernel_initializer=my_glorot_initializer,\n",
" kernel_regularizer=my_l1_regularizer,\n",
" kernel_constraint=my_positive_weights)\n",
"])\n",
"model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])\n",
"model.fit(X_train_scaled, y_train, epochs=2,\n",
" validation_data=(X_valid_scaled, y_valid))\n",
"model.save(\"my_model_with_many_custom_parts\")\n",
"model = tf.keras.models.load_model(\n",
" \"my_model_with_many_custom_parts\",\n",
" custom_objects={\n",
" \"my_l1_regularizer\": my_l1_regularizer,\n",
" \"my_positive_weights\": my_positive_weights,\n",
" \"my_glorot_initializer\": my_glorot_initializer,\n",
" \"my_softplus\": my_softplus,\n",
" }\n",
")\n",
"model.fit(X_train_scaled, y_train, epochs=2,\n",
" validation_data=(X_valid_scaled, y_valid))"
]
},
{
"cell_type": "code",
"execution_count": 95,
@@ -2311,80 +2237,6 @@
" return {\"factor\": self.factor}"
]
},
{
"cell_type": "code",
"execution_count": 96,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/2\n",
"363/363 [==============================] - 1s 1ms/step - loss: 1.4714 - mae: 0.8316 - val_loss: inf - val_mae: inf\n",
"Epoch 2/2\n",
"363/363 [==============================] - 0s 998us/step - loss: 0.8094 - mae: 0.6172 - val_loss: 2.6153 - val_mae: 0.6058\n",
"INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"INFO:tensorflow:Assets written to: my_model_with_many_custom_parts/assets\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/2\n",
"363/363 [==============================] - 1s 1ms/step - loss: 0.6333 - mae: 0.5617 - val_loss: 1.1687 - val_mae: 0.5468\n",
"Epoch 2/2\n",
"363/363 [==============================] - 0s 1ms/step - loss: 0.5570 - mae: 0.5303 - val_loss: 1.0440 - val_mae: 0.5250\n"
]
},
{
"data": {
"text/plain": [
"<keras.src.callbacks.History at 0x19b8db610>"
]
},
"execution_count": 96,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# extra code – again, show that everything works fine, this time using our\n",
"# custom regularizer class\n",
"\n",
"tf.keras.utils.set_random_seed(42)\n",
"model = tf.keras.Sequential([\n",
" tf.keras.layers.Dense(30, activation=\"relu\", kernel_initializer=\"he_normal\",\n",
" input_shape=input_shape),\n",
" tf.keras.layers.Dense(1, activation=my_softplus,\n",
" kernel_regularizer=MyL1Regularizer(0.01),\n",
" kernel_constraint=my_positive_weights,\n",
" kernel_initializer=my_glorot_initializer),\n",
"])\n",
"model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])\n",
"model.fit(X_train_scaled, y_train, epochs=2,\n",
" validation_data=(X_valid_scaled, y_valid))\n",
"model.save(\"my_model_with_many_custom_parts\")\n",
"model = tf.keras.models.load_model(\n",
" \"my_model_with_many_custom_parts\",\n",
" custom_objects={\n",
" \"MyL1Regularizer\": MyL1Regularizer,\n",
" \"my_positive_weights\": my_positive_weights,\n",
" \"my_glorot_initializer\": my_glorot_initializer,\n",
" \"my_softplus\": my_softplus,\n",
" }\n",
")\n",
"model.fit(X_train_scaled, y_train, epochs=2,\n",
" validation_data=(X_valid_scaled, y_valid))"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -2545,13 +2397,20 @@
"precision.variables"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the `reset_states()` method was renamed to `reset_state()`."
]
},
{
"cell_type": "code",
"execution_count": 104,
"metadata": {},
"outputs": [],
"source": [
"precision.reset_states()"
"precision.reset_state()"
]
},
{
@@ -2561,6 +2420,13 @@
"Creating a streaming metric:"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the `add_weight()` method's argument have a different order now, so we have to specify `name=\"total\"` and `name=\"count\"` rather than just passing the name as the first argument."
]
},
{
"cell_type": "code",
"execution_count": 105,
@@ -2572,8 +2438,8 @@
" super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
" self.threshold = threshold\n",
" self.huber_fn = create_huber(threshold)\n",
" self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
" self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
" self.total = self.add_weight(name=\"total\", initializer=\"zeros\")\n",
" self.count = self.add_weight(name=\"count\", initializer=\"zeros\")\n",
"\n",
" def update_state(self, y_true, y_pred, sample_weight=None):\n",
" sample_metrics = self.huber_fn(y_true, y_pred)\n",
@@ -2702,7 +2568,7 @@
}
],
"source": [
"m.reset_states()\n",
"m.reset_state()\n",
"m.variables"
]
},
@@ -2788,7 +2654,7 @@
}
],
"source": [
"model.save(\"my_model_with_a_custom_metric\")"
"model.save(\"my_model_with_a_custom_metric.keras\")"
]
},
{
@@ -2798,7 +2664,7 @@
"outputs": [],
"source": [
"model = tf.keras.models.load_model(\n",
" \"my_model_with_a_custom_metric\",\n",
" \"my_model_with_a_custom_metric.keras\",\n",
" custom_objects={\n",
" \"huber_fn\": create_huber(2.0),\n",
" \"HuberMetric\": HuberMetric\n",
@@ -2836,33 +2702,6 @@
"model.fit(X_train_scaled, y_train, epochs=2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`model.metrics` contains the model's loss followed by the model's metric(s), so the `HuberMetric` is `model.metrics[-1]`:"
]
},
{
"cell_type": "code",
"execution_count": 117,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2.0"
]
},
"execution_count": 117,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.metrics[-1].threshold"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -2989,7 +2828,7 @@
}
],
"source": [
"model.save(\"my_model_with_a_custom_metric_v2\")"
"model.save(\"my_model_with_a_custom_metric_v2.keras\")"
]
},
{
@@ -2998,7 +2837,7 @@
"metadata": {},
"outputs": [],
"source": [
"model = tf.keras.models.load_model(\"my_model_with_a_custom_metric_v2\",\n",
"model = tf.keras.models.load_model(\"my_model_with_a_custom_metric_v2.keras\",\n",
" custom_objects={\"HuberMetric\": HuberMetric})"
]
},
@@ -3032,28 +2871,6 @@
"model.fit(X_train_scaled, y_train, epochs=2)"
]
},
{
"cell_type": "code",
"execution_count": 126,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"2.0"
]
},
"execution_count": 126,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.metrics[-1].threshold"
]
},
{
"cell_type": "markdown",
"metadata": {},
@@ -3088,7 +2905,7 @@
],
"source": [
"# extra code – like all layers, it can be used as a function:\n",
"exponential_layer([-1., 0., 1.])"
"exponential_layer(tf.constant([-1., 0., 1.]))"
]
},
{
@@ -3215,7 +3032,7 @@
"model.fit(X_train_scaled, y_train, epochs=2,\n",
" validation_data=(X_valid_scaled, y_valid))\n",
"model.evaluate(X_test_scaled, y_test)\n",
"model.save(\"my_model_with_a_custom_layer\")"
"model.save(\"my_model_with_a_custom_layer.keras\")"
]
},
{
@@ -3246,7 +3063,7 @@
],
"source": [
"# extra code – shows how to load a model with a custom layer\n",
"model = tf.keras.models.load_model(\"my_model_with_a_custom_layer\",\n",
"model = tf.keras.models.load_model(\"my_model_with_a_custom_layer.keras\",\n",
" custom_objects={\"MyDense\": MyDense})\n",
"model.fit(X_train_scaled, y_train, epochs=2,\n",
" validation_data=(X_valid_scaled, y_valid))"
@@ -3461,6 +3278,7 @@
"class ResidualRegressor(tf.keras.Model):\n",
" def __init__(self, output_dim, **kwargs):\n",
" super().__init__(**kwargs)\n",
" self.output_dim = output_dim\n",
" self.hidden1 = tf.keras.layers.Dense(30, activation=\"relu\",\n",
" kernel_initializer=\"he_normal\")\n",
" self.block1 = ResidualBlock(2, 30)\n",
@@ -3472,7 +3290,12 @@
" for _ in range(1 + 3):\n",
" Z = self.block1(Z)\n",
" Z = self.block2(Z)\n",
" return self.out(Z)"
" return self.out(Z)\n",
"\n",
" # extra code - to be able to save and load the model below\n",
" def get_config(self):\n",
" base_config = super().get_config()\n",
" return {**base_config, \"output_dim\": self.output_dim}"
]
},
{
@@ -3507,7 +3330,7 @@
"model.compile(loss=\"mse\", optimizer=\"nadam\")\n",
"history = model.fit(X_train_scaled, y_train, epochs=2)\n",
"score = model.evaluate(X_test_scaled, y_test)\n",
"model.save(\"my_custom_model\")"
"model.save(\"my_custom_model.keras\")"
]
},
{
@@ -3542,7 +3365,7 @@
"source": [
"# extra code – the model can be loaded and you can continue training or use it\n",
"# to make predictions\n",
"model = tf.keras.models.load_model(\"my_custom_model\")\n",
"model = tf.keras.models.load_model(\"my_custom_model.keras\")\n",
"history = model.fit(X_train_scaled, y_train, epochs=2)\n",
"model.predict(X_test_scaled[:3])"
]
@@ -3575,7 +3398,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"## Losses and Metrics Based on Model Internals"
"## Losses Based on Model Internals"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the `add_metric()` method was deprecated and removed in recent TF versions. Metrics (custom or not) must now be passed to the model via the `metrics` argument of the `model.compile()` method."
]
},
{
@@ -3591,8 +3421,6 @@
" kernel_initializer=\"he_normal\")\n",
" for _ in range(5)]\n",
" self.out = tf.keras.layers.Dense(output_dim)\n",
" self.reconstruction_mean = tf.keras.metrics.Mean(\n",
" name=\"reconstruction_error\")\n",
"\n",
" def build(self, batch_input_shape):\n",
" n_inputs = batch_input_shape[-1]\n",
@@ -3605,9 +3433,6 @@
" reconstruction = self.reconstruct(Z)\n",
" recon_loss = tf.reduce_mean(tf.square(reconstruction - inputs))\n",
" self.add_loss(0.05 * recon_loss)\n",
" if training:\n",
" result = self.reconstruction_mean(recon_loss)\n",
" self.add_metric(result)\n",
" return self.out(Z)"
]
},
@@ -4202,6 +4027,13 @@
"tf.keras.utils.set_random_seed(42)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the `tf.keras.losses.mean_squared_error()` function not longer exists, you must use `tf.keras.losses.MeanSquaredError` class instead."
]
},
{
"cell_type": "code",
"execution_count": 174,
@@ -4212,7 +4044,7 @@
"batch_size = 32\n",
"n_steps = len(X_train) // batch_size\n",
"optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)\n",
"loss_fn = tf.keras.losses.mean_squared_error\n",
"loss_fn = tf.keras.losses.MeanSquaredError()\n",
"mean_loss = tf.keras.metrics.Mean()\n",
"metrics = [tf.keras.metrics.MeanAbsoluteError()]"
]
@@ -4266,7 +4098,7 @@
" print_status_bar(step, n_steps, mean_loss, metrics)\n",
"\n",
" for metric in [mean_loss] + metrics:\n",
" metric.reset_states()"
" metric.reset_state()"
]
},
{
@@ -4391,7 +4223,7 @@
" steps.set_postfix(status)\n",
"\n",
" for metric in [mean_loss] + metrics:\n",
" metric.reset_states()"
" metric.reset_state()"
]
},
{
@@ -5628,100 +5460,11 @@
"tf.keras.utils.set_random_seed(42)"
]
},
{
"cell_type": "code",
"execution_count": 232,
"metadata": {},
"outputs": [],
"source": [
"model = MyModel(dynamic=True)"
]
},
{
"cell_type": "code",
"execution_count": 233,
"metadata": {},
"outputs": [],
"source": [
"model.compile(loss=my_mse, optimizer=\"nadam\", metrics=[my_mae])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now the custom code will be called at each iteration. Let's fit, validate and evaluate with tiny datasets to avoid getting too much output:"
]
},
{
"cell_type": "code",
"execution_count": 234,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tracing MyModel.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing loss my_mse()\n",
"Tracing metric my_mae()\n",
"Tracing MyModel.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing loss my_mse()\n",
"Tracing metric my_mae()\n",
"Tracing MyModel.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing loss my_mse()\n",
"Tracing metric my_mae()\n",
"Tracing MyModel.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing loss my_mse()\n",
"Tracing metric my_mae()\n",
"Tracing MyModel.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing loss my_mse()\n",
"Tracing metric my_mae()\n",
"Tracing MyModel.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing MyDense.call()\n",
"Tracing loss my_mse()\n",
"Tracing metric my_mae()\n"
]
},
{
"data": {
"text/plain": [
"[5.545090198516846, 2.0603599548339844]"
]
},
"execution_count": 234,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model.fit(X_train_scaled[:64], y_train[:64], epochs=1,\n",
" validation_data=(X_valid_scaled[:64], y_valid[:64]), verbose=0)\n",
"model.evaluate(X_test_scaled[:64], y_test[:64], verbose=0)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Alternatively, you can compile a model with `run_eagerly=True`:"
"**Warning**: the `dynamic` argument no longer exists since Keras 3. However, you can still compile a model with `run_eagerly=True`:"
]
},
{
@@ -5826,157 +5569,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Defining custom optimizers is not very common, but in case you are one of the happy few who gets to write one, here is an example:"
]
},
{
"cell_type": "code",
"execution_count": 239,
"metadata": {},
"outputs": [],
"source": [
"class MyMomentumOptimizer(tf.keras.optimizers.Optimizer):\n",
" def __init__(self, learning_rate=0.001, momentum=0.9, name=\"MyMomentumOptimizer\", **kwargs):\n",
" \"\"\"Gradient descent with momentum optimizer.\"\"\"\n",
" super().__init__(name, **kwargs)\n",
" self._learning_rate = self._build_learning_rate(learning_rate)\n",
" self.momentum = momentum\n",
"\n",
" def build(self, var_list):\n",
" \"\"\"Initialize optimizer variables.\n",
"\n",
" Args:\n",
" var_list: list of model variables to build SGD variables on.\n",
" \"\"\"\n",
" super().build(var_list)\n",
" if getattr(self, \"_built\", False):\n",
" return\n",
" self.momentums = []\n",
" for var in var_list:\n",
" self.momentums.append(\n",
" self.add_variable_from_reference(\n",
" model_variable=var, variable_name=\"m\"\n",
" )\n",
" )\n",
" self._built = True\n",
"\n",
" def update_step(self, gradient, variable):\n",
" \"\"\"Update step given gradient and the associated model variable.\"\"\"\n",
" lr = tf.cast(self.learning_rate, variable.dtype)\n",
" m = None\n",
" var_key = self._var_key(variable)\n",
" momentum = tf.cast(self.momentum, variable.dtype)\n",
" m = self.momentums[self._index_dict[var_key]]\n",
" if m is None:\n",
" variable.assign_add(-gradient * lr)\n",
" else:\n",
" m.assign(-gradient * lr + m * momentum)\n",
" variable.assign_add(m)\n",
" \n",
" def get_config(self):\n",
" base_config = super().get_config()\n",
" print(\"Config!\")\n",
" return {\n",
" **base_config,\n",
" \"learning_rate\": self._serialize_hyperparameter(self._learning_rate),\n",
" \"momentum\": self.momentum,\n",
" }"
]
},
{
"cell_type": "code",
"execution_count": 240,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/5\n",
"363/363 [==============================] - 0s 660us/step - loss: 1.1844\n",
"Epoch 2/5\n",
"363/363 [==============================] - 0s 625us/step - loss: 0.5635\n",
"Epoch 3/5\n",
"363/363 [==============================] - 0s 609us/step - loss: 0.9703\n",
"Epoch 4/5\n",
"363/363 [==============================] - 0s 627us/step - loss: 0.5678\n",
"Epoch 5/5\n",
"363/363 [==============================] - 0s 640us/step - loss: 0.6350\n"
]
},
{
"data": {
"text/plain": [
"<keras.src.callbacks.History at 0x19c821210>"
]
},
"execution_count": 240,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"optimizer = MyMomentumOptimizer()\n",
"\n",
"tf.keras.utils.set_random_seed(42)\n",
"model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=[8])])\n",
"model.compile(loss=\"mse\", optimizer=optimizer)\n",
"model.fit(X_train_scaled, y_train, epochs=5)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's compare that to Keras's built-in momentum optimizer:"
]
},
{
"cell_type": "code",
"execution_count": 241,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/5\n",
"363/363 [==============================] - 0s 645us/step - loss: 1.1844\n",
"Epoch 2/5\n",
"363/363 [==============================] - 0s 721us/step - loss: 0.5635\n",
"Epoch 3/5\n",
"363/363 [==============================] - 0s 612us/step - loss: 0.9703\n",
"Epoch 4/5\n",
"363/363 [==============================] - 0s 625us/step - loss: 0.5678\n",
"Epoch 5/5\n",
"363/363 [==============================] - 0s 626us/step - loss: 0.6350\n"
]
},
{
"data": {
"text/plain": [
"<keras.src.callbacks.History at 0x19ea8da20>"
]
},
"execution_count": 241,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"optimizer = tf.keras.optimizers.SGD(learning_rate=0.001, momentum=0.9)\n",
"\n",
"tf.keras.utils.set_random_seed(42)\n",
"model = tf.keras.Sequential([tf.keras.layers.Dense(1, input_shape=[8])])\n",
"model.compile(loss=\"mse\", optimizer=optimizer)\n",
"model.fit(X_train_scaled, y_train, epochs=5)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Yep, we get the exact same model! 👍"
"The Keras API to create a custom optimizer has changed since Keras 3, so if you need to create one, I recommend you draw inspiration from the source code of one of Keras's builtin optimizers, such as [SGD](https://github.com/keras-team/keras/blob/b083f2e95d4d6b09f6ec22717fd1becccbdbcec1/keras/src/optimizers/sgd.py#L7)"
]
},
{
@@ -6002,7 +5595,7 @@
"3. Both `tf.range(10)` and `tf.constant(np.arange(10))` return a one-dimensional tensor containing the integers 0 to 9. However, the former uses 32-bit integers while the latter uses 64-bit integers. Indeed, TensorFlow defaults to 32 bits, while NumPy defaults to 64 bits.\n",
"4. Beyond regular tensors, TensorFlow offers several other data structures, including sparse tensors, tensor arrays, ragged tensors, queues, string tensors, and sets. The last two are actually represented as regular tensors, but TensorFlow provides special functions to manipulate them (in `tf.strings` and `tf.sets`).\n",
"5. When you want to define a custom loss function, in general you can just implement it as a regular Python function. However, if your custom loss function must support some hyperparameters (or any other state), then you should subclass the `keras.losses.Loss` class and implement the `__init__()` and `call()` methods. If you want the loss function's hyperparameters to be saved along with the model, then you must also implement the `get_config()` method.\n",
"6. Much like custom loss functions, most metrics can be defined as regular Python functions. But if you want your custom metric to support some hyperparameters (or any other state), then you should subclass the `keras.metrics.Metric` class. Moreover, if computing the metric over a whole epoch is not equivalent to computing the mean metric over all batches in that epoch (e.g., as for the precision and recall metrics), then you should subclass the `keras.metrics.Metric` class and implement the `__init__()`, `update_state()`, and `result()` methods to keep track of a running metric during each epoch. You should also implement the `reset_states()` method unless all it needs to do is reset all variables to 0.0. If you want the state to be saved along with the model, then you should implement the `get_config()` method as well.\n",
"6. Much like custom loss functions, most metrics can be defined as regular Python functions. But if you want your custom metric to support some hyperparameters (or any other state), then you should subclass the `keras.metrics.Metric` class. Moreover, if computing the metric over a whole epoch is not equivalent to computing the mean metric over all batches in that epoch (e.g., as for the precision and recall metrics), then you should subclass the `keras.metrics.Metric` class and implement the `__init__()`, `update_state()`, and `result()` methods to keep track of a running metric during each epoch. You should also implement the `reset_state()` method unless all it needs to do is reset all variables to 0.0. If you want the state to be saved along with the model, then you should implement the `get_config()` method as well.\n",
"7. You should distinguish the internal components of your model (i.e., layers or reusable blocks of layers) from the model itself (i.e., the object you will train). The former should subclass the `keras.layers.Layer` class, while the latter should subclass the `keras.models.Model` class.\n",
"8. Writing your own custom training loop is fairly advanced, so you should only do it if you really need to. Keras provides several tools to customize training without having to write a custom training loop: callbacks, custom regularizers, custom constraints, custom losses, and so on. You should use these instead of writing a custom training loop whenever possible: writing a custom training loop is more error-prone, and it will be harder to reuse the custom code you write. However, in some cases writing a custom training loop is necessary⁠—for example, if you want to use different optimizers for different parts of your neural network, like in the [Wide & Deep paper](https://homl.info/widedeep). A custom training loop can also be useful when debugging, or when trying to understand exactly how training works.\n",
"9. Custom Keras components should be convertible to TF Functions, which means they should stick to TF operations as much as possible and respect all the rules listed in Chapter 12 (in the _TF Function Rules_ section). If you absolutely need to include arbitrary Python code in a custom component, you can either wrap it in a `tf.py_function()` operation (but this will reduce performance and limit your model's portability) or set `dynamic=True` when creating the custom layer or model (or set `run_eagerly=True` when calling the model's `compile()` method).\n",
@@ -6091,6 +5684,13 @@
"Let's create one instance of each class, apply them to some data (e.g., the training set), and ensure that the difference is negligeable."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Warning**: the `mean_absolute_error()` function no longer exists. Instead, you must use the `MeanAbsoluteError` class."
]
},
{
"cell_type": "code",
"execution_count": 243,
@@ -6113,7 +5713,7 @@
"custom_layer_norm = LayerNormalization()\n",
"keras_layer_norm = tf.keras.layers.LayerNormalization()\n",
"\n",
"tf.reduce_mean(tf.keras.losses.mean_absolute_error(\n",
"tf.reduce_mean(tf.keras.losses.MeanAbsoluteError()(\n",
" keras_layer_norm(X), custom_layer_norm(X)))"
]
},
@@ -6148,7 +5748,7 @@
"custom_layer_norm.set_weights([random_alpha, random_beta])\n",
"keras_layer_norm.set_weights([random_alpha, random_beta])\n",
"\n",
"tf.reduce_mean(tf.keras.losses.mean_absolute_error(\n",
"tf.reduce_mean(tf.keras.losses.MeanAbsoluteError()(\n",
" keras_layer_norm(X), custom_layer_norm(X)))"
]
},
@@ -6343,7 +5943,7 @@
" tf.constant(y_valid, dtype=np.float32), y_pred))\n",
" steps.set_postfix(status)\n",
" for metric in [mean_loss] + metrics:\n",
" metric.reset_states()\n"
" metric.reset_state()\n"
]
},
{
@@ -6526,7 +6126,7 @@
" tf.constant(y_valid, dtype=np.float32), y_pred))\n",
" steps.set_postfix(status)\n",
" for metric in [mean_loss] + metrics:\n",
" metric.reset_states()"
" metric.reset_state()"
]
},
{
@@ -6539,7 +6139,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -6553,7 +6153,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
"version": "3.10.13"
}
},
"nbformat": 4,