Update Tutorials 0.2 (#490)
This commit is contained in:
Dario Coscia
committed by
Nicola Demo
Nicola Demo
parent
beee4cdc0b
commit
6ce0bafc2b
217
tutorials/tutorial11/tutorial.ipynb
vendored
217
tutorials/tutorial11/tutorial.ipynb
vendored
@@ -23,7 +23,6 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"## routine needed to run the notebook on Google Colab\n",
|
||||
"try:\n",
|
||||
" import google.colab\n",
|
||||
" IN_COLAB = True\n",
|
||||
@@ -43,34 +42,59 @@
|
||||
"from pina.domain import CartesianDomain\n",
|
||||
"from pina.equation import Equation, FixedValue\n",
|
||||
"\n",
|
||||
"warnings.filterwarnings('ignore')\n",
|
||||
"warnings.filterwarnings('ignore')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Define problem and solver."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# defining the ode equation\n",
|
||||
"def ode_equation(input_, output_):\n",
|
||||
"\n",
|
||||
" # computing the derivative\n",
|
||||
" u_x = grad(output_, input_, components=[\"u\"], d=[\"x\"])\n",
|
||||
"\n",
|
||||
" # extracting the u input variable\n",
|
||||
" u = output_.extract([\"u\"])\n",
|
||||
"\n",
|
||||
" # calculate the residual and return it\n",
|
||||
" return u_x - u\n",
|
||||
"\n",
|
||||
"class SimpleODE(SpatialProblem):\n",
|
||||
"\n",
|
||||
" output_variables = ['u']\n",
|
||||
" spatial_domain = CartesianDomain({'x': [0, 1]})\n",
|
||||
" output_variables = [\"u\"]\n",
|
||||
" spatial_domain = CartesianDomain({\"x\": [0, 1]})\n",
|
||||
"\n",
|
||||
" # defining the ode equation\n",
|
||||
" def ode_equation(input_, output_):\n",
|
||||
" u_x = grad(output_, input_, components=['u'], d=['x'])\n",
|
||||
" u = output_.extract(['u'])\n",
|
||||
" return u_x - u\n",
|
||||
" domains = {\n",
|
||||
" \"x0\": CartesianDomain({\"x\": 0.0}),\n",
|
||||
" \"D\": CartesianDomain({\"x\": [0, 1]}),\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # conditions to hold\n",
|
||||
" conditions = {\n",
|
||||
" 'bound_cond': Condition(domain=CartesianDomain({'x': 0.}), equation=FixedValue(1)), # We fix initial condition to value 1\n",
|
||||
" 'phys_cond': Condition(domain=CartesianDomain({'x': [0, 1]}), equation=Equation(ode_equation)), # We wrap the python equation using Equation\n",
|
||||
" \"bound_cond\": Condition(domain=\"x0\", equation=FixedValue(1.0)),\n",
|
||||
" \"phys_cond\": Condition(domain=\"D\", equation=Equation(ode_equation)),\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # defining the true solution\n",
|
||||
" def truth_solution(self, pts):\n",
|
||||
" return torch.exp(pts.extract(['x']))\n",
|
||||
" \n",
|
||||
" def solution(self, pts):\n",
|
||||
" return torch.exp(pts.extract([\"x\"]))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# sampling for training\n",
|
||||
"problem = SimpleODE()\n",
|
||||
"problem.discretise_domain(1, 'random', domains=['bound_cond'])\n",
|
||||
"problem.discretise_domain(20, 'lh', domains=['phys_cond'])\n",
|
||||
"problem.discretise_domain(1, \"random\", domains=[\"x0\"])\n",
|
||||
"problem.discretise_domain(20, \"lh\", domains=[\"D\"])\n",
|
||||
"\n",
|
||||
"# build the model\n",
|
||||
"model = FeedForward(\n",
|
||||
@@ -94,14 +118,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: True\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -136,22 +160,21 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"trainer = Trainer(solver=pinn,\n",
|
||||
" accelerator='cpu')"
|
||||
"trainer = Trainer(solver=pinn, accelerator=\"cpu\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -176,14 +199,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -192,7 +215,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 91.87it/s, v_num=8, bound_cond_loss=6.07e-5, phys_cond_loss=0.000828, train_loss=0.000889] "
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 233.15it/s, v_num=0, bound_cond_loss=1.22e-5, phys_cond_loss=0.000517, train_loss=0.000529]"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -206,14 +229,14 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 57.75it/s, v_num=8, bound_cond_loss=6.07e-5, phys_cond_loss=0.000828, train_loss=0.000889]\n"
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 137.95it/s, v_num=0, bound_cond_loss=1.22e-5, phys_cond_loss=0.000517, train_loss=0.000529]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -222,7 +245,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 79.80it/s, v_num=9, bound_cond_loss=8.63e-5, phys_cond_loss=0.00215, train_loss=0.00223] "
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 248.63it/s, v_num=1, bound_cond_loss=2.29e-5, phys_cond_loss=0.00106, train_loss=0.00108] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -236,14 +259,14 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 54.20it/s, v_num=9, bound_cond_loss=8.63e-5, phys_cond_loss=0.00215, train_loss=0.00223]\n"
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 149.06it/s, v_num=1, bound_cond_loss=2.29e-5, phys_cond_loss=0.00106, train_loss=0.00108]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -252,7 +275,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 82.98it/s, v_num=10, bound_cond_loss=2.84e-5, phys_cond_loss=0.00118, train_loss=0.00121] "
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 254.65it/s, v_num=2, bound_cond_loss=0.00029, phys_cond_loss=0.00253, train_loss=0.00282] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -266,12 +289,12 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 55.87it/s, v_num=10, bound_cond_loss=2.84e-5, phys_cond_loss=0.00118, train_loss=0.00121]\n"
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 150.72it/s, v_num=2, bound_cond_loss=0.00029, phys_cond_loss=0.00253, train_loss=0.00282]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from pytorch_lightning.loggers import TensorBoardLogger\n",
|
||||
"from lightning.pytorch.loggers import TensorBoardLogger\n",
|
||||
"\n",
|
||||
"# three run of training, by default it trains for 1000 epochs\n",
|
||||
"# we reinitialize the model each time otherwise the same parameters will be optimized\n",
|
||||
@@ -280,16 +303,18 @@
|
||||
" layers=[10, 10],\n",
|
||||
" func=torch.nn.Tanh,\n",
|
||||
" output_dimensions=len(problem.output_variables),\n",
|
||||
" input_dimensions=len(problem.input_variables)\n",
|
||||
" input_dimensions=len(problem.input_variables),\n",
|
||||
" )\n",
|
||||
" pinn = PINN(problem, model)\n",
|
||||
" trainer = Trainer(solver=pinn,\n",
|
||||
" accelerator='cpu',\n",
|
||||
" logger=TensorBoardLogger(save_dir='training_log'),\n",
|
||||
" enable_model_summary=False,\n",
|
||||
" train_size=1.0,\n",
|
||||
" val_size=0.0,\n",
|
||||
" test_size=0.0)\n",
|
||||
" trainer = Trainer(\n",
|
||||
" solver=pinn,\n",
|
||||
" accelerator=\"cpu\",\n",
|
||||
" logger=TensorBoardLogger(save_dir=\"training_log\"),\n",
|
||||
" enable_model_summary=False,\n",
|
||||
" train_size=1.0,\n",
|
||||
" val_size=0.0,\n",
|
||||
" test_size=0.0,\n",
|
||||
" )\n",
|
||||
" trainer.train()"
|
||||
]
|
||||
},
|
||||
@@ -297,7 +322,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"We can now visualize the logs by simply running `tensorboard --logdir=simpleode/` on terminal, you should obtain a webpage as the one shown below:"
|
||||
"We can now visualize the logs by simply running `tensorboard --logdir=training_log/` on terminal, you should obtain a webpage as the one shown below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -345,7 +370,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -353,12 +378,15 @@
|
||||
"from lightning.pytorch.callbacks import EarlyStopping\n",
|
||||
"import torch\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# define a simple callback\n",
|
||||
"class NaiveMetricTracker(Callback):\n",
|
||||
" def __init__(self):\n",
|
||||
" self.saved_metrics = []\n",
|
||||
"\n",
|
||||
" def on_train_epoch_end(self, trainer, __): # function called at the end of each epoch\n",
|
||||
" def on_train_epoch_end(\n",
|
||||
" self, trainer, __\n",
|
||||
" ): # function called at the end of each epoch\n",
|
||||
" self.saved_metrics.append(\n",
|
||||
" {key: value for key, value in trainer.logged_metrics.items()}\n",
|
||||
" )"
|
||||
@@ -373,14 +401,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -389,7 +417,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 90.01it/s, v_num=70, bound_cond_loss=2.14e-5, phys_cond_loss=0.000448, train_loss=0.000469] "
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 278.93it/s, v_num=0, bound_cond_loss=6.94e-5, phys_cond_loss=0.00116, train_loss=0.00123] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -403,7 +431,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 57.95it/s, v_num=70, bound_cond_loss=2.14e-5, phys_cond_loss=0.000448, train_loss=0.000469]\n"
|
||||
"Epoch 999: 100%|██████████| 1/1 [00:00<00:00, 140.62it/s, v_num=0, bound_cond_loss=6.94e-5, phys_cond_loss=0.00116, train_loss=0.00123]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -435,24 +463,24 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[{'bound_cond_loss': tensor(0.0385),\n",
|
||||
" 'phys_cond_loss': tensor(0.7217),\n",
|
||||
" 'train_loss': tensor(0.7602)},\n",
|
||||
" {'bound_cond_loss': tensor(0.0399),\n",
|
||||
" 'phys_cond_loss': tensor(0.7142),\n",
|
||||
" 'train_loss': tensor(0.7541)},\n",
|
||||
" {'bound_cond_loss': tensor(0.0413),\n",
|
||||
" 'phys_cond_loss': tensor(0.7067),\n",
|
||||
" 'train_loss': tensor(0.7480)}]"
|
||||
"[{'bound_cond_loss': tensor(0.9935),\n",
|
||||
" 'phys_cond_loss': tensor(0.0303),\n",
|
||||
" 'train_loss': tensor(1.0239)},\n",
|
||||
" {'bound_cond_loss': tensor(0.9875),\n",
|
||||
" 'phys_cond_loss': tensor(0.0293),\n",
|
||||
" 'train_loss': tensor(1.0169)},\n",
|
||||
" {'bound_cond_loss': tensor(0.9815),\n",
|
||||
" 'phys_cond_loss': tensor(0.0284),\n",
|
||||
" 'train_loss': tensor(1.0099)}]"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -472,14 +500,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -488,25 +516,28 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 4186: 100%|██████████| 1/1 [00:00<00:00, 37.06it/s, v_num=71, bound_cond_loss=1.91e-10, phys_cond_loss=3.88e-6, train_loss=3.88e-6] \n"
|
||||
"Epoch 2343: 100%|██████████| 1/1 [00:00<00:00, 64.24it/s, v_num=1, val_loss=4.79e-6, bound_cond_loss=1.15e-7, phys_cond_loss=2.33e-5, train_loss=2.34e-5] \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# ~5 mins\n",
|
||||
"\n",
|
||||
"model = FeedForward(\n",
|
||||
" layers=[10, 10],\n",
|
||||
" func=torch.nn.Tanh,\n",
|
||||
" output_dimensions=len(problem.output_variables),\n",
|
||||
" input_dimensions=len(problem.input_variables)\n",
|
||||
" )\n",
|
||||
" layers=[10, 10],\n",
|
||||
" func=torch.nn.Tanh,\n",
|
||||
" output_dimensions=len(problem.output_variables),\n",
|
||||
" input_dimensions=len(problem.input_variables),\n",
|
||||
")\n",
|
||||
"pinn = PINN(problem, model)\n",
|
||||
"trainer = Trainer(solver=pinn,\n",
|
||||
" accelerator='cpu',\n",
|
||||
" max_epochs = -1,\n",
|
||||
" enable_model_summary=False,\n",
|
||||
" callbacks=[EarlyStopping('train_loss')]) # adding a callbacks\n",
|
||||
"trainer = Trainer(\n",
|
||||
" solver=pinn,\n",
|
||||
" accelerator=\"cpu\",\n",
|
||||
" max_epochs=-1,\n",
|
||||
" enable_model_summary=False,\n",
|
||||
" val_size=0.2,\n",
|
||||
" train_size=0.8,\n",
|
||||
" test_size=0.0,\n",
|
||||
" callbacks=[EarlyStopping(\"val_loss\")],\n",
|
||||
") # adding a callbacks\n",
|
||||
"trainer.train()"
|
||||
]
|
||||
},
|
||||
@@ -540,7 +571,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -548,7 +579,7 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Seed set to 42\n",
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -557,7 +588,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 82.19it/s, v_num=72, bound_cond_loss=1.74e-6, phys_cond_loss=0.00018, train_loss=0.000182] "
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 156.69it/s, v_num=2, bound_cond_loss=1.53e-6, phys_cond_loss=0.000169, train_loss=0.000171]"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -571,8 +602,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 56.83it/s, v_num=72, bound_cond_loss=1.74e-6, phys_cond_loss=0.00018, train_loss=0.000182]\n",
|
||||
"Total training time 32.64355 s\n"
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 108.75it/s, v_num=2, bound_cond_loss=1.53e-6, phys_cond_loss=0.000169, train_loss=0.000171]\n",
|
||||
"Total training time 15.36648 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -610,7 +641,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -618,7 +649,7 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Seed set to 42\n",
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -627,7 +658,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1598: 100%|██████████| 1/1 [00:00<00:00, 70.77it/s, v_num=73, bound_cond_loss=7.01e-6, phys_cond_loss=0.000283, train_loss=0.00029] "
|
||||
"Epoch 1598: 100%|██████████| 1/1 [00:00<00:00, 224.16it/s, v_num=3, bound_cond_loss=5.7e-6, phys_cond_loss=0.000257, train_loss=0.000263] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -641,7 +672,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 62.57it/s, v_num=73, bound_cond_loss=2.74e-7, phys_cond_loss=9.51e-5, train_loss=9.54e-5] "
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 261.43it/s, v_num=3, bound_cond_loss=2.58e-7, phys_cond_loss=9.4e-5, train_loss=9.43e-5] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -655,8 +686,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 40.66it/s, v_num=73, bound_cond_loss=2.74e-7, phys_cond_loss=9.51e-5, train_loss=9.54e-5]\n",
|
||||
"Total training time 39.14717 s\n"
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 145.96it/s, v_num=3, bound_cond_loss=2.58e-7, phys_cond_loss=9.4e-5, train_loss=9.43e-5]\n",
|
||||
"Total training time 17.78182 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -697,7 +728,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
@@ -705,7 +736,7 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Seed set to 42\n",
|
||||
"GPU available: False, used: False\n",
|
||||
"GPU available: True (mps), used: False\n",
|
||||
"TPU available: False, using: 0 TPU cores\n",
|
||||
"HPU available: False, using: 0 HPUs\n"
|
||||
]
|
||||
@@ -714,7 +745,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1598: 100%|██████████| 1/1 [00:00<00:00, 69.88it/s, v_num=74, bound_cond_loss=5.16e-8, phys_cond_loss=3.54e-5, train_loss=3.54e-5] "
|
||||
"Epoch 1598: 100%|██████████| 1/1 [00:00<00:00, 251.76it/s, v_num=4, bound_cond_loss=5.98e-8, phys_cond_loss=3.88e-5, train_loss=3.88e-5] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -728,7 +759,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 73.42it/s, v_num=74, bound_cond_loss=0.000126, phys_cond_loss=0.000315, train_loss=0.000441] "
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 239.11it/s, v_num=4, bound_cond_loss=0.000333, phys_cond_loss=0.000676, train_loss=0.00101] "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -742,8 +773,8 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 47.28it/s, v_num=74, bound_cond_loss=0.000126, phys_cond_loss=0.000315, train_loss=0.000441]\n",
|
||||
"Total training time 40.19983 s\n"
|
||||
"Epoch 1999: 100%|██████████| 1/1 [00:00<00:00, 127.88it/s, v_num=4, bound_cond_loss=0.000333, phys_cond_loss=0.000676, train_loss=0.00101]\n",
|
||||
"Total training time 15.12576 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -789,7 +820,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"display_name": "pina",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@@ -803,7 +834,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.12.3"
|
||||
"version": "3.9.21"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
356
tutorials/tutorial11/tutorial.py
vendored
356
tutorials/tutorial11/tutorial.py
vendored
@@ -1,356 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# # Tutorial: PINA and PyTorch Lightning, training tips and visualizations
|
||||
#
|
||||
# [](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial11/tutorial.ipynb)
|
||||
#
|
||||
# In this tutorial, we will delve deeper into the functionality of the `Trainer` class, which serves as the cornerstone for training **PINA** [Solvers](https://mathlab.github.io/PINA/_rst/_code.html#solvers).
|
||||
#
|
||||
# The `Trainer` class offers a plethora of features aimed at improving model accuracy, reducing training time and memory usage, facilitating logging visualization, and more thanks to the amazing job done by the PyTorch Lightning team!
|
||||
#
|
||||
# Our leading example will revolve around solving the `SimpleODE` problem, as outlined in the [*Introduction to PINA for Physics Informed Neural Networks training*](https://github.com/mathLab/PINA/blob/master/tutorials/tutorial1/tutorial.ipynb). If you haven't already explored it, we highly recommend doing so before diving into this tutorial.
|
||||
#
|
||||
# Let's start by importing useful modules, define the `SimpleODE` problem and the `PINN` solver.
|
||||
|
||||
# In[1]:
|
||||
|
||||
|
||||
## routine needed to run the notebook on Google Colab
|
||||
try:
|
||||
import google.colab
|
||||
IN_COLAB = True
|
||||
except:
|
||||
IN_COLAB = False
|
||||
if IN_COLAB:
|
||||
get_ipython().system('pip install "pina-mathlab"')
|
||||
|
||||
import torch
|
||||
import warnings
|
||||
|
||||
from pina import Condition, Trainer
|
||||
from pina.solver import PINN
|
||||
from pina.model import FeedForward
|
||||
from pina.problem import SpatialProblem
|
||||
from pina.operator import grad
|
||||
from pina.domain import CartesianDomain
|
||||
from pina.equation import Equation, FixedValue
|
||||
|
||||
warnings.filterwarnings('ignore')
|
||||
|
||||
class SimpleODE(SpatialProblem):
|
||||
|
||||
output_variables = ['u']
|
||||
spatial_domain = CartesianDomain({'x': [0, 1]})
|
||||
|
||||
# defining the ode equation
|
||||
def ode_equation(input_, output_):
|
||||
u_x = grad(output_, input_, components=['u'], d=['x'])
|
||||
u = output_.extract(['u'])
|
||||
return u_x - u
|
||||
|
||||
# conditions to hold
|
||||
conditions = {
|
||||
'bound_cond': Condition(domain=CartesianDomain({'x': 0.}), equation=FixedValue(1)), # We fix initial condition to value 1
|
||||
'phys_cond': Condition(domain=CartesianDomain({'x': [0, 1]}), equation=Equation(ode_equation)), # We wrap the python equation using Equation
|
||||
}
|
||||
|
||||
# defining the true solution
|
||||
def truth_solution(self, pts):
|
||||
return torch.exp(pts.extract(['x']))
|
||||
|
||||
|
||||
# sampling for training
|
||||
problem = SimpleODE()
|
||||
problem.discretise_domain(1, 'random', domains=['bound_cond'])
|
||||
problem.discretise_domain(20, 'lh', domains=['phys_cond'])
|
||||
|
||||
# build the model
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
|
||||
# create the PINN object
|
||||
pinn = PINN(problem, model)
|
||||
|
||||
|
||||
# Till now we just followed the extact step of the previous tutorials. The `Trainer` object
|
||||
# can be initialized by simiply passing the `PINN` solver
|
||||
|
||||
# In[2]:
|
||||
|
||||
|
||||
trainer = Trainer(solver=pinn)
|
||||
|
||||
|
||||
# ## Trainer Accelerator
|
||||
#
|
||||
# When creating the trainer, **by defualt** the `Trainer` will choose the most performing `accelerator` for training which is available in your system, ranked as follow:
|
||||
# 1. [TPU](https://cloud.google.com/tpu/docs/intro-to-tpu)
|
||||
# 2. [IPU](https://www.graphcore.ai/products/ipu)
|
||||
# 3. [HPU](https://habana.ai/)
|
||||
# 4. [GPU](https://www.intel.com/content/www/us/en/products/docs/processors/what-is-a-gpu.html#:~:text=What%20does%20GPU%20stand%20for,video%20editing%2C%20and%20gaming%20applications) or [MPS](https://developer.apple.com/metal/pytorch/)
|
||||
# 5. CPU
|
||||
|
||||
# For setting manually the `accelerator` run:
|
||||
#
|
||||
# * `accelerator = {'gpu', 'cpu', 'hpu', 'mps', 'cpu', 'ipu'}` sets the accelerator to a specific one
|
||||
|
||||
# In[3]:
|
||||
|
||||
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu')
|
||||
|
||||
|
||||
# as you can see, even if in the used system `GPU` is available, it is not used since we set `accelerator='cpu'`.
|
||||
|
||||
# ## Trainer Logging
|
||||
#
|
||||
# In **PINA** you can log metrics in different ways. The simplest approach is to use the `MetricTraker` class from `pina.callbacks` as seen in the [*Introduction to PINA for Physics Informed Neural Networks training*](https://github.com/mathLab/PINA/blob/master/tutorials/tutorial1/tutorial.ipynb) tutorial.
|
||||
#
|
||||
# However, expecially when we need to train multiple times to get an average of the loss across multiple runs, `pytorch_lightning.loggers` might be useful. Here we will use `TensorBoardLogger` (more on [logging](https://lightning.ai/docs/pytorch/stable/extensions/logging.html) here), but you can choose the one you prefer (or make your own one).
|
||||
#
|
||||
# We will now import `TensorBoardLogger`, do three runs of training and then visualize the results. Notice we set `enable_model_summary=False` to avoid model summary specifications (e.g. number of parameters), set it to true if needed.
|
||||
#
|
||||
|
||||
# In[4]:
|
||||
|
||||
|
||||
from pytorch_lightning.loggers import TensorBoardLogger
|
||||
|
||||
# three run of training, by default it trains for 1000 epochs
|
||||
# we reinitialize the model each time otherwise the same parameters will be optimized
|
||||
for _ in range(3):
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
pinn = PINN(problem, model)
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu',
|
||||
logger=TensorBoardLogger(save_dir='training_log'),
|
||||
enable_model_summary=False,
|
||||
train_size=1.0,
|
||||
val_size=0.0,
|
||||
test_size=0.0)
|
||||
trainer.train()
|
||||
|
||||
|
||||
# We can now visualize the logs by simply running `tensorboard --logdir=simpleode/` on terminal, you should obtain a webpage as the one shown below:
|
||||
|
||||
# <p align=\"center\">
|
||||
# <img src="logging.png" alt=\"Logging API\" width=\"400\"/>
|
||||
# </p>
|
||||
|
||||
# as you can see, by default, **PINA** logs the losses which are shown in the progress bar, as well as the number of epochs. You can always insert more loggings by either defining a **callback** ([more on callbacks](https://lightning.ai/docs/pytorch/stable/extensions/callbacks.html)), or inheriting the solver and modify the programs with different **hooks** ([more on hooks](https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#hooks)).
|
||||
|
||||
# ## Trainer Callbacks
|
||||
|
||||
# Whenever we need to access certain steps of the training for logging, do static modifications (i.e. not changing the `Solver`) or updating `Problem` hyperparameters (static variables), we can use `Callabacks`. Notice that `Callbacks` allow you to add arbitrary self-contained programs to your training. At specific points during the flow of execution (hooks), the Callback interface allows you to design programs that encapsulate a full set of functionality. It de-couples functionality that does not need to be in **PINA** `Solver`s.
|
||||
# Lightning has a callback system to execute them when needed. Callbacks should capture NON-ESSENTIAL logic that is NOT required for your lightning module to run.
|
||||
#
|
||||
# The following are best practices when using/designing callbacks.
|
||||
#
|
||||
# * Callbacks should be isolated in their functionality.
|
||||
# * Your callback should not rely on the behavior of other callbacks in order to work properly.
|
||||
# * Do not manually call methods from the callback.
|
||||
# * Directly calling methods (eg. on_validation_end) is strongly discouraged.
|
||||
# * Whenever possible, your callbacks should not depend on the order in which they are executed.
|
||||
#
|
||||
# We will try now to implement a naive version of `MetricTraker` to show how callbacks work. Notice that this is a very easy application of callbacks, fortunately in **PINA** we already provide more advanced callbacks in `pina.callbacks`.
|
||||
#
|
||||
# <!-- Suppose we want to log the accuracy on some validation poit -->
|
||||
|
||||
# In[5]:
|
||||
|
||||
|
||||
from lightning.pytorch.callbacks import Callback
|
||||
from lightning.pytorch.callbacks import EarlyStopping
|
||||
import torch
|
||||
|
||||
# define a simple callback
|
||||
class NaiveMetricTracker(Callback):
|
||||
def __init__(self):
|
||||
self.saved_metrics = []
|
||||
|
||||
def on_train_epoch_end(self, trainer, __): # function called at the end of each epoch
|
||||
self.saved_metrics.append(
|
||||
{key: value for key, value in trainer.logged_metrics.items()}
|
||||
)
|
||||
|
||||
|
||||
# Let's see the results when applyed to the `SimpleODE` problem. You can define callbacks when initializing the `Trainer` by the `callbacks` argument, which expects a list of callbacks.
|
||||
|
||||
# In[6]:
|
||||
|
||||
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
pinn = PINN(problem, model)
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu',
|
||||
logger=True,
|
||||
callbacks=[NaiveMetricTracker()], # adding a callbacks
|
||||
enable_model_summary=False,
|
||||
train_size=1.0,
|
||||
val_size=0.0,
|
||||
test_size=0.0)
|
||||
trainer.train()
|
||||
|
||||
|
||||
# We can easily access the data by calling `trainer.callbacks[0].saved_metrics` (notice the zero representing the first callback in the list given at initialization).
|
||||
|
||||
# In[7]:
|
||||
|
||||
|
||||
trainer.callbacks[0].saved_metrics[:3] # only the first three epochs
|
||||
|
||||
|
||||
# PyTorch Lightning also has some built in `Callbacks` which can be used in **PINA**, [here an extensive list](https://lightning.ai/docs/pytorch/stable/extensions/callbacks.html#built-in-callbacks).
|
||||
#
|
||||
# We can for example try the `EarlyStopping` routine, which automatically stops the training when a specific metric converged (here the `train_loss`). In order to let the training keep going forever set `max_epochs=-1`.
|
||||
|
||||
# In[8]:
|
||||
|
||||
|
||||
# ~5 mins
|
||||
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
pinn = PINN(problem, model)
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu',
|
||||
max_epochs = -1,
|
||||
enable_model_summary=False,
|
||||
callbacks=[EarlyStopping('train_loss')]) # adding a callbacks
|
||||
trainer.train()
|
||||
|
||||
|
||||
# As we can see the model automatically stop when the logging metric stopped improving!
|
||||
|
||||
# ## Trainer Tips to Boost Accuracy, Save Memory and Speed Up Training
|
||||
#
|
||||
# Untill now we have seen how to choose the right `accelerator`, how to log and visualize the results, and how to interface with the program in order to add specific parts of code at specific points by `callbacks`.
|
||||
# Now, we well focus on how boost your training by saving memory and speeding it up, while mantaining the same or even better degree of accuracy!
|
||||
#
|
||||
#
|
||||
# There are several built in methods developed in PyTorch Lightning which can be applied straight forward in **PINA**, here we report some:
|
||||
#
|
||||
# * [Stochastic Weight Averaging](https://pytorch.org/blog/pytorch-1.6-now-includes-stochastic-weight-averaging/) to boost accuracy
|
||||
# * [Gradient Clippling](https://deepgram.com/ai-glossary/gradient-clipping) to reduce computational time (and improve accuracy)
|
||||
# * [Gradient Accumulation](https://lightning.ai/docs/pytorch/stable/common/optimization.html#id3) to save memory consumption
|
||||
# * [Mixed Precision Training](https://lightning.ai/docs/pytorch/stable/common/optimization.html#id3) to save memory consumption
|
||||
#
|
||||
# We will just demonstrate how to use the first two, and see the results compared to a standard training.
|
||||
# We use the [`Timer`](https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.callbacks.Timer.html#lightning.pytorch.callbacks.Timer) callback from `pytorch_lightning.callbacks` to take the times. Let's start by training a simple model without any optimization (train for 2000 epochs).
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
from lightning.pytorch.callbacks import Timer
|
||||
from lightning.pytorch import seed_everything
|
||||
|
||||
# setting the seed for reproducibility
|
||||
seed_everything(42, workers=True)
|
||||
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
|
||||
pinn = PINN(problem, model)
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu',
|
||||
deterministic=True, # setting deterministic=True ensure reproducibility when a seed is imposed
|
||||
max_epochs = 2000,
|
||||
enable_model_summary=False,
|
||||
callbacks=[Timer()]) # adding a callbacks
|
||||
trainer.train()
|
||||
print(f'Total training time {trainer.callbacks[0].time_elapsed("train"):.5f} s')
|
||||
|
||||
|
||||
# Now we do the same but with StochasticWeightAveraging
|
||||
|
||||
# In[10]:
|
||||
|
||||
|
||||
from lightning.pytorch.callbacks import StochasticWeightAveraging
|
||||
|
||||
# setting the seed for reproducibility
|
||||
seed_everything(42, workers=True)
|
||||
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
pinn = PINN(problem, model)
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu',
|
||||
deterministic=True,
|
||||
max_epochs = 2000,
|
||||
enable_model_summary=False,
|
||||
callbacks=[Timer(),
|
||||
StochasticWeightAveraging(swa_lrs=0.005)]) # adding StochasticWeightAveraging callbacks
|
||||
trainer.train()
|
||||
print(f'Total training time {trainer.callbacks[0].time_elapsed("train"):.5f} s')
|
||||
|
||||
|
||||
# As you can see, the training time does not change at all! Notice that around epoch `1600`
|
||||
# the scheduler is switched from the defalut one `ConstantLR` to the Stochastic Weight Average Learning Rate (`SWALR`).
|
||||
# This is because by default `StochasticWeightAveraging` will be activated after `int(swa_epoch_start * max_epochs)` with `swa_epoch_start=0.7` by default. Finally, the final `mean_loss` is lower when `StochasticWeightAveraging` is used.
|
||||
#
|
||||
# We will now now do the same but clippling the gradient to be relatively small.
|
||||
|
||||
# In[11]:
|
||||
|
||||
|
||||
# setting the seed for reproducibility
|
||||
seed_everything(42, workers=True)
|
||||
|
||||
model = FeedForward(
|
||||
layers=[10, 10],
|
||||
func=torch.nn.Tanh,
|
||||
output_dimensions=len(problem.output_variables),
|
||||
input_dimensions=len(problem.input_variables)
|
||||
)
|
||||
pinn = PINN(problem, model)
|
||||
trainer = Trainer(solver=pinn,
|
||||
accelerator='cpu',
|
||||
max_epochs = 2000,
|
||||
enable_model_summary=False,
|
||||
gradient_clip_val=0.1, # clipping the gradient
|
||||
callbacks=[Timer(),
|
||||
StochasticWeightAveraging(swa_lrs=0.005)])
|
||||
trainer.train()
|
||||
print(f'Total training time {trainer.callbacks[0].time_elapsed("train"):.5f} s')
|
||||
|
||||
|
||||
# As we can see we by applying gradient clipping we were able to even obtain lower error!
|
||||
#
|
||||
# ## What's next?
|
||||
#
|
||||
# Now you know how to use efficiently the `Trainer` class **PINA**! There are multiple directions you can go now:
|
||||
#
|
||||
# 1. Explore training times on different devices (e.g.) `TPU`
|
||||
#
|
||||
# 2. Try to reduce memory cost by mixed precision training and gradient accumulation (especially useful when training Neural Operators)
|
||||
#
|
||||
# 3. Benchmark `Trainer` speed for different precisions.
|
||||
Reference in New Issue
Block a user