diff --git a/docs/source/tutorials/tutorial21/tutorial.html b/docs/source/tutorials/tutorial21/tutorial.html index 8db72c2..da4fe65 100644 --- a/docs/source/tutorials/tutorial21/tutorial.html +++ b/docs/source/tutorials/tutorial21/tutorial.html @@ -7544,7 +7544,7 @@ a.anchor-link {
-

Tutorial: Introductory Tutorial: Supervised Learning with PINA

Open In Colab

+

Tutorial: Introductory Tutorial: Supervised Learning with PINA

Open In Colab

⚠️ Before starting:

We assume you are already familiar with the concepts covered in the Getting started with PINA tutorials. If not, we strongly recommend reviewing them before exploring this advanced topic.

@@ -7681,7 +7681,7 @@ $$

@@ -7867,7 +7867,7 @@ The Graph Neural Operator leverages Graph Neural Networ @@ -7894,12 +7894,12 @@ The Graph Neural Operator leverages Graph Neural Networ
@@ -7979,7 +7979,7 @@ var element = document.getElementById('8def35e0-b355-4036-86b2-3324b38c5d38'); @@ -8009,6 +8009,6 @@ var element = document.getElementById('8def35e0-b355-4036-86b2-3324b38c5d38'); diff --git a/docs/source/tutorials/tutorial8/tutorial.html b/docs/source/tutorials/tutorial8/tutorial.html index 1ab0d87..70cb91e 100644 --- a/docs/source/tutorials/tutorial8/tutorial.html +++ b/docs/source/tutorials/tutorial8/tutorial.html @@ -7544,7 +7544,7 @@ a.anchor-link { @@ -7795,7 +7795,7 @@ a.anchor-link { @@ -7840,12 +7840,12 @@ a.anchor-link { @@ -8018,8 +8018,8 @@ var element = document.getElementById('9053264c-f81c-48a8-b2b8-d1aa4d506e4f'); @@ -8111,7 +8111,7 @@ var element = document.getElementById('9053264c-f81c-48a8-b2b8-d1aa4d506e4f'); @@ -8148,6 +8148,6 @@ var element = document.getElementById('9053264c-f81c-48a8-b2b8-d1aa4d506e4f'); diff --git a/tutorials/tutorial1/tutorial.py b/tutorials/tutorial1/tutorial.py index ee9afca..94ac090 100644 --- a/tutorials/tutorial1/tutorial.py +++ b/tutorials/tutorial1/tutorial.py @@ -1,15 +1,15 @@ #!/usr/bin/env python # coding: utf-8 -# # Tutorial: Introductory Tutorial: Physics Informed Neural Networks with PINA +# # Tutorial: Introductory Tutorial: Physics Informed Neural Networks with PINA # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial1/tutorial.ipynb) -# +# # > ##### ⚠️ ***Before starting:*** # > We assume you are already familiar with the concepts covered in the [Getting started with PINA](https://mathlab.github.io/PINA/_tutorial.html#getting-started-with-pina) tutorials. If not, we strongly recommend reviewing them before exploring this advanced topic. -# +# # In this tutorial, we will demonstrate a typical use case of **PINA** for Physics Informed Neural Network (PINN) training. We will cover the basics of training a PINN with PINA, if you want to go further into PINNs look at our dedicated [tutorials](https://mathlab.github.io/PINA/_tutorial.html#physics-informed-neural-networks) on the topic. -# +# # Let's start by importing the useful modules: # In[ ]: @@ -43,9 +43,9 @@ warnings.filterwarnings("ignore") # ## Build the problem -# +# # We will use a simple Ordinary Differential Equation as pedagogical example: -# +# # $$ # \begin{equation} # \begin{cases} @@ -54,9 +54,9 @@ warnings.filterwarnings("ignore") # \end{cases} # \end{equation} # $$ -# -# with the analytical solution $u(x) = e^x$. -# +# +# with the analytical solution $u(x) = e^x$. +# # The PINA problem is easly written as: # In[2]: @@ -100,8 +100,8 @@ problem.discretise_domain(1, "lh", domains=["x0"]) problem.discretise_domain(20, "lh", domains=["D"]) -# ## Generate data -# +# ## Generate data +# # Data for training can come in form of direct numerical simulation results, or points in the domains. In case we perform unsupervised learning, we just need the collocation points for training, i.e. points where we want to evaluate the neural network. Sampling point in **PINA** is very easy, here we show three examples using the `.discretise_domain` method of the `AbstractProblem` class. # In[4]: @@ -144,18 +144,18 @@ _ = plt.legend() # ## Easily solve a Physics Problem with three step pipeline # Once the problem is defined and the data is generated, we can move on to modeling. This process consists of three key steps: -# +# # **Choosing a Model** # - Select a neural network architecture. You can use the model we provide in the `pina.model` module (see [here](https://mathlab.github.io/PINA/_rst/_code.html#models) for a full list), or define a custom PyTorch module (more on this [here](https://pytorch.org/docs/stable/notes/modules.html)). -# +# # **Choosing a PINN Solver & Defining the Trainer** # * Use a Physics Informed solver from `pina.solver` module to solve the problem using the specified model. We have already implemented most State-Of-The-Arte solvers for you, [have a look](https://mathlab.github.io/PINA/_rst/_code.html#solvers) if interested. Today we will use the standard `PINN` solver. -# +# # **Training** # * Train the model with the [`Trainer`](https://mathlab.github.io/PINA/_rst/trainer.html) class. The Trainer class provides powerful features to enhance model accuracy, optimize training time and memory, and simplify logging and visualization, thanks to PyTorch Lightning's excellent work, see [our dedicated tutorial](https://mathlab.github.io/PINA/tutorial11/tutorial.html) for further details. By default, training metrics (e.g., MSE error) are logged using a lightning logger (CSVLogger). If you prefer manual tracking, use `pina.callback.MetricTracker`. -# +# # Let's cover all steps one by one! -# +# # First we build the model, in this case a FeedForward neural network, with two layers of size 10 and hyperbolic tangent activation: # In[7]: @@ -171,7 +171,7 @@ model = FeedForward( # Then we build the solver. The Physics-Informed Neural Network (`PINN`) solver class needs to be initialised with a `model` and a specific `problem` to be solved. They also take extra arguments, as the optimizer, scheduler, loss type and weighting for the different conditions which are all set to their defualt values. -# +# # >##### 💡***Bonus tip:*** # > All physics solvers in PINA can handle both forward and inverse problems without requiring any changes to the model or solver structure! See [our tutorial](https://mathlab.github.io/PINA/tutorial7/tutorial.html) of inverse problems for more infos. @@ -184,10 +184,10 @@ pinn = PINN(problem, model, TorchOptimizer(torch.optim.RAdam, lr=0.005)) # Finally, we train the model using the Trainer API. The trainer offers various options to customize your training, refer to the official documentation for details. Here, we highlight the `MetricTracker` from `pina.callback`, which helps track metrics during training. In order to train just call the `.train()` method. -# +# # > ##### ⚠️ ***Important Note:*** # > In PINA you can log metrics in different ways. The simplest approach is to use the `MetricTraker` class from `pina.callbacks` as we will see today. However, expecially when we need to train multiple times to get an average of the loss across multiple runs, we suggest to use `lightning.pytorch.loggers` (see [here](https://lightning.ai/docs/pytorch/stable/extensions/logging.html) for reference). -# +# # In[9]: @@ -218,7 +218,7 @@ trainer.train() trainer.logged_metrics -# By using `matplotlib` we can also do some qualitative plots of the solution. +# By using `matplotlib` we can also do some qualitative plots of the solution. # In[11]: @@ -249,17 +249,17 @@ plt.yscale("log") # ## What's Next? -# +# # Congratulations on completing the introductory tutorial on Physics-Informed Training! Now that you have a solid foundation, here are several exciting directions you can explore: -# +# # 1. **Experiment with Training Duration & Network Architecture**: Try different training durations and tweak the network architecture to optimize performance. -# +# # 2. **Explore Other Models in `pina.model`**: Check out other models available in `pina.model` or design your own custom PyTorch module to suit your needs. -# +# # 3. **Run Training on a GPU**: Speed up your training by running on a GPU and compare the performance improvements. -# +# # 4. **Test Various Solvers**: Explore and evaluate different solvers to assess their performance on various types of problems. -# +# # 5. **... and many more!**: The possibilities are vast! Continue experimenting with advanced configurations, solvers, and other features in PINA. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial10/tutorial.py b/tutorials/tutorial10/tutorial.py index ac03fe6..b06445f 100644 --- a/tutorials/tutorial10/tutorial.py +++ b/tutorials/tutorial10/tutorial.py @@ -2,12 +2,12 @@ # coding: utf-8 # # Tutorial: Solving the Kuramoto–Sivashinsky Equation with Averaging Neural Operator -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial10/tutorial.ipynb) -# -# +# +# # In this tutorial, we will build a Neural Operator using the **`AveragingNeuralOperator`** model and the **`SupervisedSolver`**. By the end of this tutorial, you will be able to train a Neural Operator to learn the operator for time-dependent PDEs. -# +# # Let's start by importing the necessary modules. # In[ ]: @@ -24,8 +24,12 @@ if IN_COLAB: get_ipython().system('pip install "pina-mathlab[tutorial]"') # get the data get_ipython().system('mkdir "data"') - get_ipython().system('wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial10/data/Data_KS.mat" -O "data/Data_KS.mat"') - get_ipython().system('wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial10/data/Data_KS2.mat" -O "data/Data_KS2.mat"') + get_ipython().system( + 'wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial10/data/Data_KS.mat" -O "data/Data_KS.mat"' + ) + get_ipython().system( + 'wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial10/data/Data_KS2.mat" -O "data/Data_KS2.mat"' + ) import torch import matplotlib.pyplot as plt @@ -41,36 +45,36 @@ warnings.filterwarnings("ignore") # ## Data Generation -# +# # In this tutorial, we will focus on solving the **Kuramoto-Sivashinsky (KS)** equation, a fourth-order nonlinear PDE. The equation is given by: -# +# # $$ # \frac{\partial u}{\partial t}(x,t) = -u(x,t)\frac{\partial u}{\partial x}(x,t) - \frac{\partial^{4}u}{\partial x^{4}}(x,t) - \frac{\partial^{2}u}{\partial x^{2}}(x,t). # $$ -# +# # In this equation, $x \in \Omega = [0, 64]$ represents a spatial location, and $t \in \mathbb{T} = [0, 50]$ represents time. The function $u(x, t)$ is the value of the function at each point in space and time, with $u(x, t) \in \mathbb{R}$. We denote the solution space as $\mathbb{U}$, where $u \in \mathbb{U}$. -# +# # We impose Dirichlet boundary conditions on the derivative of $u$ at the boundary of the domain $\partial \Omega$: -# +# # $$ # \frac{\partial u}{\partial x}(x,t) = 0 \quad \forall (x,t) \in \partial \Omega \times \mathbb{T}. # $$ -# +# # The initial conditions are sampled from a distribution over truncated Fourier series with random coefficients $\{A_k, \ell_k, \phi_k\}_k$, as follows: -# +# # $$ # u(x,0) = \sum_{k=1}^N A_k \sin\left(2 \pi \frac{\ell_k x}{L} + \phi_k\right), # $$ -# +# # where: # - $A_k \in [-0.4, -0.3]$, # - $\ell_k = 2$, # - $\phi_k = 2\pi \quad \forall k=1,\dots,N$. -# -# We have already generated data for different initial conditions. The goal is to build a Neural Operator that, given $u(x,t)$, outputs $u(x,t+\delta)$, where $\delta$ is a fixed time step. -# +# +# We have already generated data for different initial conditions. The goal is to build a Neural Operator that, given $u(x,t)$, outputs $u(x,t+\delta)$, where $\delta$ is a fixed time step. +# # We will cover the Neural Operator architecture later, but for now, let’s start by importing the data. -# +# # **Note:** # The numerical integration is obtained using a pseudospectral method for spatial derivative discretization and implicit Runge-Kutta 5 for temporal dynamics. @@ -102,7 +106,7 @@ print(f" shape solution: {sol_train.shape}") # - `B` is the batch size (i.e., how many initial conditions we sample), # - `N` is the number of points in the mesh (which is the product of the discretization in $x$ times the one in $t$), # - `D` is the dimension of the problem (in this case, we have three variables: $[u, t, x]$). -# +# # We are now going to plot some trajectories! # In[4]: @@ -166,36 +170,36 @@ plot_trajectory( # As we can see, as time progresses, the solution becomes chaotic, making it very difficult to learn! We will now focus on building a Neural Operator using the `SupervisedSolver` class to tackle this problem. -# +# # ## Averaging Neural Operator -# +# # We will build a neural operator $\texttt{NO}$, which takes the solution at time $t=0$ for any $x\in\Omega$, the time $t$ at which we want to compute the solution, and gives back the solution to the KS equation $u(x, t)$. Mathematically: -# +# # $$ # \texttt{NO}_\theta : \mathbb{U} \rightarrow \mathbb{U}, # $$ -# +# # such that -# +# # $$ # \texttt{NO}_\theta[u(t=0)](x, t) \rightarrow u(x, t). # $$ -# +# # There are many ways to approximate the following operator, for example, by using a 2D [FNO](https://mathlab.github.io/PINA/_rst/model/fourier_neural_operator.html) (for regular meshes), a [DeepOnet](https://mathlab.github.io/PINA/_rst/model/deeponet.html), [Continuous Convolutional Neural Operator](https://mathlab.github.io/PINA/_rst/model/block/convolution.html), or [MIONet](https://mathlab.github.io/PINA/_rst/model/mionet.html). In this tutorial, we will use the *Averaging Neural Operator* presented in [*The Nonlocal Neural Operator: Universal Approximation*](https://arxiv.org/abs/2304.13221), which is a [Kernel Neural Operator](https://mathlab.github.io/PINA/_rst/model/kernel_neural_operator.html) with an integral kernel: -# +# # $$ # K(v) = \sigma\left(Wv(x) + b + \frac{1}{|\Omega|}\int_\Omega v(y)dy\right) # $$ -# +# # where: -# +# # * $v(x) \in \mathbb{R}^{\rm{emb}}$ is the update for a function $v$, with $\mathbb{R}^{\rm{emb}}$ being the embedding (hidden) size. # * $\sigma$ is a non-linear activation function. # * $W \in \mathbb{R}^{\rm{emb} \times \rm{emb}}$ is a tunable matrix. # * $b \in \mathbb{R}^{\rm{emb}}$ is a tunable bias. -# +# # In PINA, many Kernel Neural Operators are already implemented. The modular components of the [Kernel Neural Operator](https://mathlab.github.io/PINA/_rst/model/kernel_neural_operator.html) class allow you to create new ones by composing base kernel layers. -# +# # **Note:** We will use the already built class `AveragingNeuralOperator`. As a constructive exercise, try to use the [KernelNeuralOperator](https://mathlab.github.io/PINA/_rst/model/kernel_neural_operator.html) class to build a kernel neural operator from scratch. You might employ the different layers that we have in PINA, such as [FeedForward](https://mathlab.github.io/PINA/_rst/model/feed_forward.html) and [AveragingNeuralOperator](https://mathlab.github.io/PINA/_rst/model/average_neural_operator.html) layers. # In[5]: @@ -222,9 +226,9 @@ model = AveragingNeuralOperator( # Super easy! Notice that we use the `SIREN` activation function, which is discussed in more detail in the paper [Implicit Neural Representations with Periodic Activation Functions](https://arxiv.org/abs/2006.09661). -# +# # ## Solving the KS problem -# +# # We will now focus on solving the KS equation using the `SupervisedSolver` class and the `AveragingNeuralOperator` model. As done in the [FNO tutorial](https://github.com/mathLab/PINA/blob/master/tutorials/tutorial5/tutorial.ipynb), we now create the Neural Operator problem class with `SupervisedProblem`. # In[6]: @@ -267,7 +271,7 @@ plot_trajectory( ) -# As we can see, we can obtain nice results considering the small training time and the difficulty of the problem! +# As we can see, we can obtain nice results considering the small training time and the difficulty of the problem! # Let's take a look at the training and testing error: # In[8]: @@ -293,13 +297,13 @@ with torch.no_grad(): # As we can see, the error is pretty small, which aligns with the observations from the previous plots. # ## What's Next? -# +# # You have completed the tutorial on solving time-dependent PDEs using Neural Operators in **PINA**. Great job! Here are some potential next steps you can explore: -# +# # 1. **Train the network for longer or with different layer sizes**: Experiment with various configurations, such as adjusting the number of layers or hidden dimensions, to further improve accuracy and observe the impact on performance. -# +# # 2. **Use a more challenging dataset**: Try using the more complex dataset [Data_KS2.mat](dat/Data_KS2.mat) where $A_k \in [-0.5, 0.5]$, $\ell_k \in [1, 2, 3]$, and $\phi_k \in [0, 2\pi]$ for a more difficult task. This dataset may require longer training and testing. -# +# # 3. **... and many more...**: Explore other models, such as the [FNO](https://mathlab.github.io/PINA/_rst/models/fno.html), [DeepOnet](https://mathlab.github.io/PINA/_rst/models/deeponet.html), or implement your own operator using the [KernelNeuralOperator](https://mathlab.github.io/PINA/_rst/models/base_no.html) class to compare performance and find the best model for your task. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial11/tutorial.py b/tutorials/tutorial11/tutorial.py index 3cd917f..597f93c 100644 --- a/tutorials/tutorial11/tutorial.py +++ b/tutorials/tutorial11/tutorial.py @@ -3,15 +3,15 @@ # # Tutorial: Introduction to `Trainer` class # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial11/tutorial.ipynb) -# -# In this tutorial, we will delve deeper into the functionality of the `Trainer` class, which serves as the cornerstone for training **PINA** [Solvers](https://mathlab.github.io/PINA/_rst/_code.html#solvers). -# +# +# In this tutorial, we will delve deeper into the functionality of the `Trainer` class, which serves as the cornerstone for training **PINA** [Solvers](https://mathlab.github.io/PINA/_rst/_code.html#solvers). +# # The `Trainer` class offers a plethora of features aimed at improving model accuracy, reducing training time and memory usage, facilitating logging visualization, and more thanks to the amazing job done by the PyTorch Lightning team! -# +# # Our leading example will revolve around solving a simple regression problem where we want to approximate the following function with a Neural Net model $\mathcal{M}_{\theta}$: # $$y = x^3$$ # by having only a set of $20$ observations $\{x_i, y_i\}_{i=1}^{20}$, with $x_i \sim\mathcal{U}[-3, 3]\;\;\forall i\in(1,\dots,20)$. -# +# # Let's start by importing useful modules! # In[ ]: @@ -70,16 +70,16 @@ trainer = Trainer(solver=solver) # ## Trainer Accelerator -# +# # When creating the `Trainer`, **by default** the most performing `accelerator` for training which is available in your system will be chosen, ranked as follows: # 1. [TPU](https://cloud.google.com/tpu/docs/intro-to-tpu) # 2. [IPU](https://www.graphcore.ai/products/ipu) # 3. [HPU](https://habana.ai/) # 4. [GPU](https://www.intel.com/content/www/us/en/products/docs/processors/what-is-a-gpu.html#:~:text=What%20does%20GPU%20stand%20for,video%20editing%2C%20and%20gaming%20applications) or [MPS](https://developer.apple.com/metal/pytorch/) # 5. CPU -# +# # For setting manually the `accelerator` run: -# +# # * `accelerator = {'gpu', 'cpu', 'hpu', 'mps', 'cpu', 'ipu'}` sets the accelerator to a specific one # In[15]: @@ -91,11 +91,11 @@ trainer = Trainer(solver=solver, accelerator="cpu") # As you can see, even if a `GPU` is available on the system, it is not used since we set `accelerator='cpu'`. # ## Trainer Logging -# +# # In **PINA** you can log metrics in different ways. The simplest approach is to use the `MetricTracker` class from `pina.callbacks`, as seen in the [*Introduction to Physics Informed Neural Networks training*](https://github.com/mathLab/PINA/blob/master/tutorials/tutorial1/tutorial.ipynb) tutorial. -# +# # However, especially when we need to train multiple times to get an average of the loss across multiple runs, `lightning.pytorch.loggers` might be useful. Here we will use `TensorBoardLogger` (more on [logging](https://lightning.ai/docs/pytorch/stable/extensions/logging.html) here), but you can choose the one you prefer (or make your own one). -# +# # We will now import `TensorBoardLogger`, do three runs of training, and then visualize the results. Notice we set `enable_model_summary=False` to avoid model summary specifications (e.g. number of parameters); set it to `True` if needed. # In[17]: @@ -133,21 +133,21 @@ for _ in range(3): #

# As you can see, by default, **PINA** logs the losses which are shown in the progress bar, as well as the number of epochs. You can always insert more loggings by either defining a **callback** ([more on callbacks](https://lightning.ai/docs/pytorch/stable/extensions/callbacks.html)), or inheriting the solver and modifying the programs with different **hooks** ([more on hooks](https://lightning.ai/docs/pytorch/stable/common/lightning_module.html#hooks)). -# +# # ## Trainer Callbacks -# +# # Whenever we need to access certain steps of the training for logging, perform static modifications (i.e. not changing the `Solver`), or update `Problem` hyperparameters (static variables), we can use **Callbacks**. Notice that **Callbacks** allow you to add arbitrary self-contained programs to your training. At specific points during the flow of execution (hooks), the Callback interface allows you to design programs that encapsulate a full set of functionality. It de-couples functionality that does not need to be in **PINA** `Solver`s. -# +# # Lightning has a callback system to execute them when needed. **Callbacks** should capture NON-ESSENTIAL logic that is NOT required for your lightning module to run. -# +# # The following are best practices when using/designing callbacks: -# +# # * Callbacks should be isolated in their functionality. # * Your callback should not rely on the behavior of other callbacks in order to work properly. # * Do not manually call methods from the callback. # * Directly calling methods (e.g., on_validation_end) is strongly discouraged. # * Whenever possible, your callbacks should not depend on the order in which they are executed. -# +# # We will try now to implement a naive version of `MetricTraker` to show how callbacks work. Notice that this is a very easy application of callbacks, fortunately in **PINA** we already provide more advanced callbacks in `pina.callbacks`. # In[18]: @@ -172,7 +172,7 @@ class NaiveMetricTracker(Callback): # Let's see the results when applied to the problem. You can define **callbacks** when initializing the `Trainer` by using the `callbacks` argument, which expects a list of callbacks. -# +# # In[19]: @@ -206,8 +206,8 @@ trainer.train() trainer.callbacks[0].saved_metrics[:3] # only the first three epochs -# PyTorch Lightning also has some built-in `Callbacks` which can be used in **PINA**, [here is an extensive list](https://lightning.ai/docs/pytorch/stable/extensions/callbacks.html#built-in-callbacks). -# +# PyTorch Lightning also has some built-in `Callbacks` which can be used in **PINA**, [here is an extensive list](https://lightning.ai/docs/pytorch/stable/extensions/callbacks.html#built-in-callbacks). +# # We can, for example, try the `EarlyStopping` routine, which automatically stops the training when a specific metric converges (here the `train_loss`). In order to let the training keep going forever, set `max_epochs=-1`. # In[22]: @@ -237,17 +237,17 @@ trainer.train() # As we can see the model automatically stop when the logging metric stopped improving! # ## Trainer Tips to Boost Accuracy, Save Memory and Speed Up Training -# +# # Until now we have seen how to choose the right `accelerator`, how to log and visualize the results, and how to interface with the program in order to add specific parts of code at specific points via `callbacks`. # Now, we will focus on how to boost your training by saving memory and speeding it up, while maintaining the same or even better degree of accuracy! -# +# # There are several built-in methods developed in PyTorch Lightning which can be applied straightforward in **PINA**. Here we report some: -# +# # * [Stochastic Weight Averaging](https://pytorch.org/blog/pytorch-1.6-now-includes-stochastic-weight-averaging/) to boost accuracy # * [Gradient Clipping](https://deepgram.com/ai-glossary/gradient-clipping) to reduce computational time (and improve accuracy) # * [Gradient Accumulation](https://lightning.ai/docs/pytorch/stable/common/optimization.html#id3) to save memory consumption # * [Mixed Precision Training](https://lightning.ai/docs/pytorch/stable/common/optimization.html#id3) to save memory consumption -# +# # We will just demonstrate how to use the first two and see the results compared to standard training. # We use the [`Timer`](https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.callbacks.Timer.html#lightning.pytorch.callbacks.Timer) callback from `pytorch_lightning.callbacks` to track the times. Let's start by training a simple model without any optimization (train for 500 epochs). @@ -312,7 +312,7 @@ print(f'Total training time {trainer.callbacks[0].time_elapsed("train"):.5f} s') # As you can see, the training time does not change at all! Notice that around epoch 350 # the scheduler is switched from the defalut one `ConstantLR` to the Stochastic Weight Average Learning Rate (`SWALR`). # This is because by default `StochasticWeightAveraging` will be activated after `int(swa_epoch_start * max_epochs)` with `swa_epoch_start=0.7` by default. Finally, the final `train_loss` is lower when `StochasticWeightAveraging` is used. -# +# # We will now do the same but clippling the gradient to be relatively small. # In[25]: @@ -341,18 +341,18 @@ print(f'Total training time {trainer.callbacks[0].time_elapsed("train"):.5f} s') # As we can see, by applying gradient clipping, we were able to achieve even lower error! -# +# # ## What's Next? -# +# # Now you know how to use the `Trainer` class efficiently in **PINA**! There are several directions you can explore next: -# +# # 1. **Explore Training on Different Devices**: Test training times on various devices (e.g., `TPU`) to compare performance. -# +# # 2. **Reduce Memory Costs**: Experiment with mixed precision training and gradient accumulation to optimize memory usage, especially when training Neural Operators. -# +# # 3. **Benchmark `Trainer` Speed**: Benchmark the training speed of the `Trainer` class for different precisions to identify potential optimizations. -# +# # 4. **...and many more!**: Consider expanding to **multi-GPU** setups or other advanced configurations for large-scale training. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). -# +# diff --git a/tutorials/tutorial12/tutorial.py b/tutorials/tutorial12/tutorial.py index 383869c..55be71f 100644 --- a/tutorials/tutorial12/tutorial.py +++ b/tutorials/tutorial12/tutorial.py @@ -2,18 +2,18 @@ # coding: utf-8 # # Tutorial: Introduction to PINA `Equation` class -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial12/tutorial.ipynb) -# -# +# +# # In this tutorial, we will explore how to use the `Equation` class in **PINA**. We will focus on how to leverage this class, along with its inherited subclasses, to enforce residual minimization in **Physics-Informed Neural Networks (PINNs)**. -# +# # By the end of this guide, you'll understand how to integrate physical laws and constraints directly into your model training, ensuring that the solution adheres to the underlying differential equations. -# -# +# +# # ## Example: The Burgers 1D equation # We will start implementing the viscous Burgers 1D problem Class, described as follows: -# +# # $$ # \begin{equation} # \begin{cases} @@ -23,9 +23,9 @@ # \end{cases} # \end{equation} # $$ -# +# # where we set $ \nu = \frac{0.01}{\pi}$. -# +# # In the class that models this problem we will see in action the `Equation` class and one of its inherited classes, the `FixedValue` class. # In[ ]: @@ -78,11 +78,11 @@ def initial_condition(input_, output_): # - `input_`: A tensor with respect to which the operator is computed. # - `components`: The names of the output variables for which the operator is evaluated. # - `d`: The names of the variables with respect to which the operator is computed. -# +# # Each differential operator has its **fast** version, which performs no internal checks on input and output tensors. For these methods, the user is always required to specify both ``components`` and ``d`` as lists of strings. -# +# # Let's define now the problem! -# +# # > **👉 Do you want to learn more on Problems? Check the dedicated [tutorial](https://mathlab.github.io/PINA/tutorial16/tutorial.html) to learn how to build a Problem from scratch.** # In[ ]: @@ -118,25 +118,25 @@ class Burgers1D(TimeDependentProblem, SpatialProblem): } -# The `Equation` class takes as input a function (in this case it happens twice, with `initial_condition` and `burger_equation`) which computes a residual of an equation, such as a PDE. In a problem class such as the one above, the `Equation` class with such a given input is passed as a parameter in the specified `Condition`. -# +# The `Equation` class takes as input a function (in this case it happens twice, with `initial_condition` and `burger_equation`) which computes a residual of an equation, such as a PDE. In a problem class such as the one above, the `Equation` class with such a given input is passed as a parameter in the specified `Condition`. +# # The `FixedValue` class takes as input a value of the same dimensions as the output functions. This class can be used to enforce a fixed value for a specific condition, such as Dirichlet boundary conditions, as demonstrated in our example. -# -# Once the equations are set as above in the problem conditions, the PINN solver will aim to minimize the residuals described in each equation during the training phase. -# +# +# Once the equations are set as above in the problem conditions, the PINN solver will aim to minimize the residuals described in each equation during the training phase. +# # ### Available classes of equations: # - `FixedGradient` and `FixedFlux`: These work analogously to the `FixedValue` class, where we can enforce a constant value on the gradient or the divergence of the solution, respectively. # - `Laplace`: This class can be used to enforce that the Laplacian of the solution is zero. # - `SystemEquation`: This class allows you to enforce multiple conditions on the same subdomain by passing a list of residual equations defined in the problem. -# +# # ## Defining a new Equation class # `Equation` classes can also be inherited to define a new class. For example, we can define a new class `Burgers1D` to represent the Burgers equation. During the class call, we can pass the viscosity parameter $\nu$: -# +# # ```python # class Burgers1D(Equation): # def __init__(self, nu): # self.nu = nu -# +# # def equation(self, input_, output_): # ... # ``` @@ -207,14 +207,14 @@ class Burgers1D(TimeDependentProblem, SpatialProblem): # ## What's Next? -# +# # Congratulations on completing the `Equation` class tutorial of **PINA**! As we've seen, you can build new classes that inherit from `Equation` to store more complex equations, such as the 1D Burgers equation, by simply passing the characteristic coefficients of the problem. -# +# # From here, you can: -# +# # - **Define Additional Complex Equation Classes**: Create your own equation classes, such as `SchrodingerEquation`, `NavierStokesEquation`, etc. # - **Define More `FixedOperator` Classes**: Implement operators like `FixedCurl`, `FixedDivergence`, and others for more advanced simulations. # - **Integrate Custom Equations and Operators**: Combine your custom equations and operators into larger systems for more complex simulations. # - **and many more!**: Explore for example different residual minimization techniques to improve the performance and accuracy of your models. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial13/tutorial.py b/tutorials/tutorial13/tutorial.py index d456eb6..3a69cde 100644 --- a/tutorials/tutorial13/tutorial.py +++ b/tutorials/tutorial13/tutorial.py @@ -2,13 +2,13 @@ # coding: utf-8 # # Tutorial: Learning Multiscale PDEs Using Fourier Feature Networks -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial13/tutorial.ipynb) -# +# # This tutorial demonstrates how to solve a PDE with multiscale behavior using Physics-Informed Neural Networks (PINNs), as discussed in [*On the Eigenvector Bias of Fourier Feature Networks: From Regression to Solving Multi-Scale PDEs with Physics-Informed Neural Networks*](https://doi.org/10.1016/j.cma.2021.113938). -# +# # Let’s begin by importing the necessary libraries. -# +# # In[ ]: @@ -41,30 +41,30 @@ warnings.filterwarnings("ignore") # ## Multiscale Problem -# +# # We begin by presenting the problem, which is also discussed in Section 2 of [*On the Eigenvector Bias of Fourier Feature Networks: From Regression to Solving Multi-Scale PDEs with Physics-Informed Neural Networks*](https://doi.org/10.1016/j.cma.2021.113938). The one-dimensional Poisson problem we aim to solve is mathematically defined as: -# +# # \begin{equation} # \begin{cases} # \Delta u(x) + f(x) = 0 \quad x \in [0,1], \\ # u(x) = 0 \quad x \in \partial[0,1], # \end{cases} # \end{equation} -# +# # We define the solution as: -# +# # $$ # u(x) = \sin(2\pi x) + 0.1 \sin(50\pi x), # $$ -# +# # which leads to the corresponding force term: -# +# # $$ # f(x) = (2\pi)^2 \sin(2\pi x) + 0.1 (50 \pi)^2 \sin(50\pi x). # $$ -# +# # While this example is simple and pedagogical, it's important to note that the solution exhibits low-frequency behavior in the macro-scale and high-frequency behavior in the micro-scale. This characteristic is common in many practical scenarios. -# +# # Below is the implementation of the `Poisson` problem as described mathematically above. # > **👉 We have a dedicated [tutorial](https://mathlab.github.io/PINA/tutorial16/tutorial.html) to teach how to build a Problem from scratch — have a look if you're interested!** @@ -112,12 +112,12 @@ problem.discretise_domain(128, "grid", domains=["phys_cond"]) problem.discretise_domain(1, "grid", domains=["bound_cond0", "bound_cond1"]) -# A standard PINN approach would involve fitting the model using a Feed Forward (fully connected) Neural Network. For a conventional fully-connected neural network, it is relatively easy to approximate a function $u$, given sufficient data inside the computational domain. -# +# A standard PINN approach would involve fitting the model using a Feed Forward (fully connected) Neural Network. For a conventional fully-connected neural network, it is relatively easy to approximate a function $u$, given sufficient data inside the computational domain. +# # However, solving high-frequency or multi-scale problems presents significant challenges to PINNs, especially when the number of data points is insufficient to capture the different scales effectively. -# +# # Below, we run a simulation using both the `PINN` solver and the self-adaptive `SAPINN` solver, employing a [`FeedForward`](https://mathlab.github.io/PINA/_modules/pina/model/feed_forward.html#FeedForward) model. -# +# # In[3]: @@ -182,10 +182,10 @@ plt.figure() plot_solution(sapinn, "Self Adaptive PINN solution") -# We can clearly observe that neither of the two solvers has successfully learned the solution. -# The issue is not with the optimization strategy (i.e., the solver), but rather with the model used to solve the problem. +# We can clearly observe that neither of the two solvers has successfully learned the solution. +# The issue is not with the optimization strategy (i.e., the solver), but rather with the model used to solve the problem. # A simple `FeedForward` network struggles to handle multiscale problems, especially when there are not enough collocation points to capture the different scales effectively. -# +# # Next, let's compute the $l_2$ relative error for both the `PINN` and `SAPINN` solutions: # In[5]: @@ -205,20 +205,20 @@ print( # Which is indeed very high! -# +# # ## Fourier Feature Embedding in PINA # Fourier Feature Embedding is a technique used to transform the input features, aiding the network in learning multiscale variations in the output. It was first introduced in [*On the Eigenvector Bias of Fourier Feature Networks: From Regression to Solving Multi-Scale PDEs with Physics-Informed Neural Networks*](https://doi.org/10.1016/j.cma.2021.113938), where it demonstrated excellent results for multiscale problems. -# +# # The core idea behind Fourier Feature Embedding is to map the input $\mathbf{x}$ into an embedding $\tilde{\mathbf{x}}$, defined as: -# +# # $$ # \tilde{\mathbf{x}} = \left[\cos\left( \mathbf{B} \mathbf{x} \right), \sin\left( \mathbf{B} \mathbf{x} \right)\right], # $$ -# +# # where $\mathbf{B}_{ij} \sim \mathcal{N}(0, \sigma^2)$. This simple operation allows the network to learn across multiple scales! -# +# # In **PINA**, we have already implemented this feature as a `layer` called [`FourierFeatureEmbedding`](https://mathlab.github.io/PINA/_rst/layers/fourier_embedding.html). Below, we will build the *Multi-scale Fourier Feature Architecture*. In this architecture, multiple Fourier feature embeddings (initialized with different $\sigma$ values) are applied to the input coordinates. These embeddings are then passed through the same fully-connected neural network, and the outputs are concatenated with a final linear layer. -# +# # In[6]: @@ -243,7 +243,7 @@ class MultiscaleFourierNet(torch.nn.Module): return self.final_layer(torch.cat([e1, e2], dim=-1)) -# We will train the `MultiscaleFourierNet` using the `PINN` solver. +# We will train the `MultiscaleFourierNet` using the `PINN` solver. # Feel free to experiment with other PINN variants as well, such as `SAPINN`, `GPINN`, `CompetitivePINN`, and others, to see how they perform on this multiscale problem. # In[7]: @@ -278,17 +278,17 @@ print( # It is clear that the network has learned the correct solution, with a very low error. Of course, longer training and a more expressive neural network could further improve the results! -# +# # ## What's Next? -# +# # Congratulations on completing the one-dimensional Poisson tutorial of **PINA** using `FourierFeatureEmbedding`! There are many potential next steps you can explore: -# +# # 1. **Train the network longer or with different layer sizes**: Experiment with different configurations to improve accuracy. -# +# # 2. **Understand the role of `sigma` in `FourierFeatureEmbedding`**: The original paper provides insightful details on the impact of `sigma`. It's a good next step to dive deeper into its effect. -# +# # 3. **Implement the *Spatio-temporal Multi-scale Fourier Feature Architecture***: Code this architecture for a more complex, time-dependent PDE (refer to Section 3 of the original paper). -# +# # 4. **...and many more!**: There are countless directions to further explore, from testing on different problems to refining the model architecture. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial14/tutorial.py b/tutorials/tutorial14/tutorial.py index 3855e6e..7c0b94d 100644 --- a/tutorials/tutorial14/tutorial.py +++ b/tutorials/tutorial14/tutorial.py @@ -2,11 +2,11 @@ # coding: utf-8 # # Tutorial: Learning Bifurcating PDE Solutions with Physics-Informed Deep Ensembles -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial14/tutorial.ipynb) -# +# # This tutorial demonstrates how to use the Deep Ensemble Physics Informed Network (DeepEnsemblePINN) to learn PDEs exhibiting bifurcating behavior, as discussed in [*Learning and Discovering Multiple Solutions Using Physics-Informed Neural Networks with Random Initialization and Deep Ensemble*](https://arxiv.org/abs/2503.06320). -# +# # Let’s begin by importing the necessary libraries. # In[ ]: @@ -41,62 +41,62 @@ warnings.filterwarnings("ignore") # ## Deep Ensemble -# +# # Deep Ensemble methods improve model performance by leveraging the diversity of predictions generated by multiple neural networks trained on the same problem. Each network in the ensemble is trained independently—typically with different weight initializations or even slight variations in the architecture or data sampling. By combining their outputs (e.g., via averaging or majority voting), ensembles reduce overfitting, increase robustness, and improve generalization. -# +# # This approach allows the ensemble to capture different perspectives of the problem, leading to more accurate and reliable predictions. -# +# #

# PINA Workflow #

-# +# # The image above illustrates a Deep Ensemble setup, where multiple models attempt to predict the text from an image. While individual models may make errors (e.g., predicting "PONY" instead of "PINA"), combining their outputs—such as taking the majority vote—often leads to the correct result. This ensemble effect improves reliability by mitigating the impact of individual model biases. -# -# +# +# # ## Deep Ensemble Physics-Informed Networks -# +# # In the context of Physics-Informed Neural Networks (PINNs), Deep Ensembles help the network discover different branches or multiple solutions of a PDE that exhibits bifurcating behavior. -# +# # By training a diverse set of models with different initializations, Deep Ensemble methods overcome the limitations of single-initialization models, which may converge to only one of the possible solutions. This approach is particularly useful when the solution space of the problem contains multiple valid physical states or behaviors. -# -# +# +# # ## The Bratu Problem -# +# # In this tutorial, we'll train a `DeepEnsemblePINN` solver to solve a bifurcating ODE known as the **Bratu problem**. The ODE is given by: -# +# # $$ # \frac{d^2u}{dt^2} + \lambda e^u = 0, \quad t \in (0, 1) # $$ -# +# # with boundary conditions: -# +# # $$ # u(0) = u(1) = 0, # $$ -# +# # where $\lambda > 0$ is a scalar parameter. The analytical solutions to the 1D Bratu problem can be expressed as: -# +# # $$ # u(t, \alpha) = 2 \log\left(\frac{\cosh(\alpha)}{\cosh(\alpha(1 - 2t))}\right), # $$ -# +# # where $\alpha$ satisfies: -# +# # $$ # \cosh(\alpha) - 2\sqrt{2}\alpha = 0. # $$ -# +# # When $\lambda < 3.513830719$, the equation admits two solutions $\alpha_1$ and $\alpha_2$, which correspond to two distinct solutions of the original ODE: $u_1$ and $u_2$. -# +# # In this tutorial, we set $\lambda = 1$, which leads to: -# +# # - $\alpha_1 \approx 0.37929$ # - $\alpha_2 \approx 2.73468$ -# +# # We first write the problem class, we do not write the boundary conditions as we will hard impose them. -# +# # > **👉 We have a dedicated [tutorial](https://mathlab.github.io/PINA/tutorial16/tutorial.html) to teach how to build a Problem — have a look if you're interested!** -# +# # > **👉 We have a dedicated [tutorial](https://mathlab.github.io/PINA/tutorial3/tutorial.html) to teach how to impose hard constraints — have a look if you're interested!** # In[80]: @@ -135,11 +135,11 @@ problem.discretise_domain(n=101, mode="grid", domains="interior") # ## Defining the Deep Ensemble Models -# +# # Now that the problem setup is complete, we move on to creating an **ensemble of models**. Each ensemble member will be a standard `FeedForward` neural network, wrapped inside a custom `Model` class. -# +# # Each model's weights are initialized using a **normal distribution** with mean 0 and standard deviation 2. This random initialization is crucial to promote diversity across the ensemble members, allowing the models to converge to potentially different solutions of the PDE. -# +# # The final ensemble is simply a **list of PyTorch models**, which we will later pass to the `DeepEnsemblePINN` # In[81]: @@ -179,15 +179,15 @@ with torch.no_grad(): # As you can see we get different output since the neural networks are initialized differently. -# +# # ## Training with `DeepEnsemblePINN` -# +# # Now that everything is ready, we can train the models using the `DeepEnsemblePINN` solver! 🎯 -# +# # This solver is constructed by combining multiple neural network models that all aim to solve the same PDE. Each model $\mathcal{M}_{i \in \{1, \dots, 10\}}$ in the ensemble contributes a unique perspective due to different random initializations. -# +# # This diversity allows the ensemble to **capture multiple branches or bifurcating solutions** of the problem, making it especially powerful for PDEs like the Bratu problem. -# +# # Once the `DeepEnsemblePINN` solver is defined with all the models, we train them using the `Trainer` class, as with any other solver in **PINA**. We also build a callback to store the value of `u(0.5)` during training iterations. # In[83]: @@ -243,11 +243,11 @@ with torch.no_grad(): # As you can see, different networks in the ensemble converge to different values pf $u(0.5)$ — this means we can actually **spot the bifurcation** in the solution space! -# +# # This is a powerful demonstration of how **Deep Ensemble Physics-Informed Neural Networks** are capable of learning **multiple valid solutions** of a PDE that exhibits bifurcating behavior. -# +# # We can also visualize the ensemble predictions to better observe the multiple branches: -# +# # In[88]: @@ -270,13 +270,13 @@ with torch.no_grad(): # ## What's Next? -# +# # You have completed the tutorial on deep ensemble PINNs for bifurcating PDEs, well don! There are many potential next steps you can explore: -# +# # 1. **Train the network longer or with different hyperparameters**: Experiment with different configurations of the single model, you can compose an ensemble by also stacking models with different layers, activation, ... to improve accuracy. -# +# # 2. **Solve more complex problems**: The original paper provides very complex problems that can be solved with PINA, we suggest you to try implement and solve them! -# +# # 3. **...and many more!**: There are countless directions to further explore, for example, what does it happen when you vary the network initialization hyperparameters? -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial15/tutorial.py b/tutorials/tutorial15/tutorial.py index 787bd51..b25b670 100644 --- a/tutorials/tutorial15/tutorial.py +++ b/tutorials/tutorial15/tutorial.py @@ -2,15 +2,15 @@ # coding: utf-8 # # Tutorial: Chemical Properties Prediction with Graph Neural Networks -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial15/tutorial.ipynb) -# -# In this tutorial we will use **Graph Neural Networks** (GNNs) for chemical properties prediction. Chemical properties prediction involves estimating or determining the physical, chemical, or biological characteristics of molecules based on their structure. -# +# +# In this tutorial we will use **Graph Neural Networks** (GNNs) for chemical properties prediction. Chemical properties prediction involves estimating or determining the physical, chemical, or biological characteristics of molecules based on their structure. +# # Molecules can naturally be represented as graphs, where atoms serve as the nodes and chemical bonds as the edges connecting them. This graph-based structure makes GNNs a great fit for predicting chemical properties. -# +# # In the tutorial we will use the [QM9 dataset](https://pytorch-geometric.readthedocs.io/en/latest/generated/torch_geometric.datasets.QM9.html#torch_geometric.datasets.QM9) from Pytorch Geometric. The dataset contains small molecules, each consisting of up to 29 atoms, with every atom having a corresponding 3D position. Each atom is also represented by a five-dimensional one-hot encoded vector that indicates the atom type (H, C, N, O, F). -# +# # First of all, let's start by importing useful modules! # In[ ]: @@ -42,7 +42,7 @@ warnings.filterwarnings("ignore") # ## Download Data and create the Problem # We download the dataset and save the molecules as a list of `Data` objects (`input_`), where each element contains one molecule encoded in a graph structure. The corresponding target properties (`target_`) are listed below: -# +# # | Target | Property | Description | Unit | # |--------|----------------------------------|-----------------------------------------------------------------------------------|---------------------------------------------| # | 0 | $\mu$ | Dipole moment | $D$ | @@ -64,7 +64,7 @@ warnings.filterwarnings("ignore") # | 16 | $A$ | Rotational constant | $GHz$ | # | 17 | $B$ | Rotational constant | $GHz$ | # | 18 | $C$ | Rotational constant | $GHz$ | -# +# # In[ ]: @@ -92,9 +92,9 @@ problem = SupervisedProblem(input_=input_, output_=target_) # ## Build the Model -# +# # To predict molecular properties, we will construct a simple Convolutional Graph Neural Network using the [`GCNConv`]() module from PyG. While this tutorial focuses on a straightforward model, more advanced architectures—such as Equivariant Networks—could potentially yield better performance. Please note that this tutorial serves only for demonstration purposes. -# +# # **Importantly** notice that in the `forward` pass we pass a data object as input, and unpack inside the graph attributes. This is the only requirement in **PINA** to use graphs and solvers together. # In[4]: @@ -118,7 +118,7 @@ class GNN(torch.nn.Module): # ## Train the Model -# +# # Now that the problem is created and the model is built, we can train the model using the [`SupervisedSolver`](https://mathlab.github.io/PINA/_rst/solver/supervised.html), which is the solver for standard supervised learning task. We will optimize the Maximum Absolute Error and test on the same metric. In the [`Trainer`](https://mathlab.github.io/PINA/_rst/trainer.html) class we specify the optimization hyperparameters. # In[5]: @@ -153,7 +153,7 @@ _ = trainer.test() # We observe that the model achieves an average error of approximately 0.4 MAE across all property predictions. This error is an average, but we can also inspect the error for each individual property prediction. -# +# # To do this, we need access to the test dataset, which can be retrieved from the trainer's datamodule. Each datamodule contains both the dataloader and dataset objects. For the dataset, we can use the [`get_all_data()`](https://mathlab.github.io/PINA/_rst/data/dataset.html#pina.data.dataset.PinaDataset.get_all_data) method. This function returns the entire dataset as a dictionary, where the keys represent the Condition names, and the values are dictionaries containing input and target tensors. # In[7]: @@ -301,15 +301,15 @@ plt.show() # By looking more into details, we can see that $A$ is not predicted that well, but the small values of the quantity lead to a lower MAE than the other properties. From the plot we can see that the atomatization energies, free energy and enthalpy are the predicted properties with higher correlation with the true chemical properties. # ## What's Next? -# +# # Congratulations on completing the tutorial on chemical properties prediction with **PINA**! Now that you've got the basics, there are several exciting directions to explore: -# +# # 1. **Train the network for longer or with different layer sizes**: Experiment with various configurations to see how the network's accuracy improves. -# +# # 2. **Use a different network**: For example, Equivariant Graph Neural Networks (EGNNs) have shown great results on molecular tasks by leveraging group symmetries. If you're interested, check out [*E(n) Equivariant Graph Neural Networks*](https://arxiv.org/abs/2102.09844) for more details. -# +# # 3. **What if the input is time-dependent?**: For example, predicting force fields in Molecular Dynamics simulations. In PINA, you can predict force fields with ease, as it's still a supervised learning task. If this interests you, have a look at [*Machine Learning Force Fields*](https://pubs.acs.org/doi/10.1021/acs.chemrev.0c01111). -# +# # 4. **...and many more!**: The possibilities are vast, including exploring new architectures, working with larger datasets, and applying this framework to more complex systems. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial16/tutorial.py b/tutorials/tutorial16/tutorial.py index 1460d16..81c9900 100644 --- a/tutorials/tutorial16/tutorial.py +++ b/tutorials/tutorial16/tutorial.py @@ -2,16 +2,16 @@ # coding: utf-8 # # Tutorial: How to build a Problem in PINA -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial16/tutorial.ipynb) -# +# # In this tutorial, we will demonstrate how to build a **Problem** in **PINA** using a toy example. The tutorial will cover the following topics: -# +# # - **Building a Problem**: Learn how to construct a problem using the built-in PINA classes. # - **Generating Data for Physics-Informed Training**: Understand how to generate the necessary data for training. # - **Exploring the `problem.zoo` Module**: Get familiar with the `problem.zoo` module, which collects pre-built problems for easy use. -# +# # By the end of this tutorial, you'll be able to write **data-driven** or **differential problems** in **PINA** and prepare them for model training! # In[ ]: @@ -37,31 +37,31 @@ warnings.filterwarnings("ignore") # ## Build a PINA problem # In **PINA**, defining a problem is done by creating a Python `class` that inherits from one or more problem classes, such as `SpatialProblem`, `TimeDependentProblem`, or `ParametricProblem`, depending on the nature of the problem. We refer to the `model` as the object that solves the problem, e.g., a **Neural Network**. -# +# # We can have two types of problems: # 1. ***Data-Driven Problems***: The model is trained using data, such as in classification networks or autoencoders. # 2. **&Physics-Driven Problems***: The model is trained using physical laws representing the problem, such as in **PINNs**. -# Let's start by building the first type, the data driven type. -# +# Let's start by building the first type, the data driven type. +# # ### Data driven modelling # In data-driven modelling, we always have an **input** and a **target**. The model's objective is to reconstruct the target from the input. Examples include: # - Image reconstruction (perturbed image as input, clear image as target) # - Classification (e.g., input: molecule, target: chemical properties) -# +# # To build a data-driven problem in **PINA**, you can inherit from the `AbstractProblem` class. Below is an example of a regression problem where the input is a scalar value `x` and the target is a scalar value `y`. -# +# # ```python # from pina.problem import AbstractProblem -# +# # class SupervisedProblem(AbstractProblem): -# +# # input_variables = ['x'] # output_variables = ['y'] -# +# # # other stuff ... # ``` # Observe that we define `input_variables` and `output_variables` as lists of symbols. This is because, in PINA, `torch.Tensors` can be labeled (see [`LabelTensor`](https://mathlab.github.io/PINA/_rst/label_tensor.html)), providing maximum flexibility for tensor manipulation. If you prefer to use regular tensors, you can simply set these to ``None``. -# +# # To specify the input and target data, you need to use the [`Condition`](https://mathlab.github.io/PINA/_rst/condition/condition.html) interface. A condition defines the constraints (such as physical equations, boundary conditions, etc.) that must be satisfied within the problem. Once the condition is applied, the full problem is outlined below: # In[3]: @@ -92,10 +92,10 @@ problem = SupervisedProblem() # You can define as many conditions as needed, and the model will attempt to minimize all of them simultaneously! You can access the data in various ways: -# +# # - `problem.conditions[''].input`, `problem.conditions[''].output` – Access the input and output data for the specified condition ``. # - `problem.input_pts` – Access the input points for all conditions. -# +# # To ensure that the problem is ready, you can check if all domains have been discretized, meaning all conditions have input points available to pass to the model: # In[4]: @@ -109,7 +109,7 @@ problem.are_all_domains_discretised # ### Simple Ordinary Differential Equation # What if we don't have data but we know the physical laws that define the data? Then physics-informed training is the solution! As an example, consider the following Ordinary Differential Equation (ODE): -# +# # $$ # \begin{equation} # \begin{cases} @@ -118,39 +118,39 @@ problem.are_all_domains_discretised # \end{cases} # \end{equation} # $$ -# +# # with the analytical solution $u(x) = e^x$. This problem is a spatial problem because the ODE depends only on the spatial variable $x\in(0,1)$. In PINA, differential problems are categorized by their nature, e.g.: # * `SpatialProblem` $\rightarrow$ a differential equation with spatial variable(s) # * `TimeDependentProblem` $\rightarrow$ a time-dependent differential equation with temporal variable(s) # * `ParametricProblem` $\rightarrow$ a parametrized differential equation with parametric variable(s) # * `InverseProblem` $\rightarrow$ this is a more advanced topic, see [this tutorial](https://mathlab.github.io/PINA/tutorial7/tutorial.html) for more details. -# +# # In our case, the physical ODE inherits from the `SpatialProblem` class, since only spatial variables define the ODE. -# +# # ```python # class SimpleODE(SpatialProblem): -# +# # output_variables = ['u'] # spatial_domain = CartesianDomain{'x': [0, 1]}) -# +# # # other stuff ... # ``` -# +# # What if our equation is was also time-dependent, e.g. Partial Differential Equations (PDE)? In this case, our `class` will inherit from both `SpatialProblem` and `TimeDependentProblem`: -# -# +# +# # ```python # class TimeSpaceODE(SpatialProblem, TimeDependentProblem): -# +# # output_variables = ["u"] # spatial_domain = CartesianDomain({"x": [0, 1]}) # temporal_domain = CartesianDomain({"t": [0, 1]}) -# +# # # other stuff ... # ``` -# +# # Differently from data-driven problems, differential-problems need to specify the domain type. If you look at our ODE definition, the spatial varibale $x$ is defined in the interval $(0,1)$, and accordingly the `spatial_domain` is a `CartesianDomain` with the input variable `x` in `[0,1]`. To know more about the Domain class see the [related tutorial](https://mathlab.github.io/PINA/tutorial6/tutorial.html). Different problems require different domain, here below we summarize the relevant ones: -# +# # | Problem Type | Required Domain | # |-------------------------|--------------------------------| # | `SpatialProblem` | `spatial_domain` | @@ -203,25 +203,25 @@ class SimpleODE(SpatialProblem): return torch.exp(pts.extract(["x"])) -# As you can see, we implemented the `ode_equation` function which given the model ouput and input returns the equation residual. These residuals are the ones minimized during PINN optimization (for more on PINN see [the related tutorials](https://mathlab.github.io/PINA/_tutorial.html#physics-informed-neural-networks)). -# +# As you can see, we implemented the `ode_equation` function which given the model ouput and input returns the equation residual. These residuals are the ones minimized during PINN optimization (for more on PINN see [the related tutorials](https://mathlab.github.io/PINA/_tutorial.html#physics-informed-neural-networks)). +# # How are the residuals computed? -# Givem the output we perform differential operation using the [operator modulus](https://mathlab.github.io/PINA/_rst/operator.html). It is pretty intuitive, each differential operator takes the following inputs: -# - A tensor on which the operator is applied. -# - A tensor with respect to which the operator is computed. -# - The names of the output variables for which the operator is evaluated. +# Givem the output we perform differential operation using the [operator modulus](https://mathlab.github.io/PINA/_rst/operator.html). It is pretty intuitive, each differential operator takes the following inputs: +# - A tensor on which the operator is applied. +# - A tensor with respect to which the operator is computed. +# - The names of the output variables for which the operator is evaluated. # - The names of the variables with respect to which the operator is computed. -# We also have a `fast` version of differential operators, where no checks are performed. This can be used to boost performances, once you know the standard ones are doing their job. -# +# We also have a `fast` version of differential operators, where no checks are performed. This can be used to boost performances, once you know the standard ones are doing their job. +# # Notice that we do not pass directly a `python` function, but an `Equation` object, which is initialized with the `python` function. This is done so that all the computations and internal checks are done inside **PINA**, see [the related tutorials](https://mathlab.github.io/PINA/tutorial12/tutorial.html) for more. -# +# # Once we have defined the function, we need to tell the neural network where these methods are to be applied. To do so, we use again the `Condition` class. In the `Condition` class, we pass the location points and the equation we want minimized on those points. -# +# # Finally, it's possible to define a `solution` function, which can be useful if we want to plot the results and see how the real solution compares to the expected (true) solution. Notice that the `solution` function is a method of the `Problem` class, but it is not mandatory for problem definition. -# +# # ## Generate data for Physical Problems -# +# # When training physics based models, data can come in form of direct numerical simulation results (tensors, graph), or points in the domains which need to be sampled. In case we perform unsupervised learning, we just need the collocation points for training, i.e. points where we want to evaluate the neural network. Sampling point in **PINA** is very easy. But first, let's check if the domains are dicsretized by using the `are_all_domains_discretised` method. # In[16]: @@ -232,7 +232,7 @@ problem.are_all_domains_discretised # This is false becase the input points are not available (we need to discretize!). If you call `problem.input_points` at this stage you will get an error due to point missing in the condition. -# +# # ```bash # >>> problem.input_pts # ``` @@ -241,13 +241,13 @@ problem.are_all_domains_discretised # KeyError Traceback (most recent call last) # Cell In[32], line 1 # ----> 1 problem.input_pts -# +# # File ~/GitHub/PINA/pina/problem/abstract_problem.py:78, in AbstractProblem.input_pts(self) # 76 to_return[cond_name] = cond.input # 77 elif hasattr(cond, "domain"): # ---> 78 to_return[cond_name] = self._discretised_domains[cond.domain] # 79 return to_return -# +# # KeyError: 'x0' # ``` @@ -300,9 +300,9 @@ plt.legend() # ## The Problem Zoo module -# +# # In PINA many problems are already implemented for you in the [Problem Zoo module](https://mathlab.github.io/PINA/_rst/_code.html#problems-zoo). For example, the supervised problem at the beginning of the tutorial is implemented in [`SupervisedProblem`](https://mathlab.github.io/PINA/_rst/problem/zoo/supervised_problem.html)! -# +# # Let's see now a physics based example, the advection equation # In[30]: @@ -324,13 +324,13 @@ print( # ## What's Next? -# +# # Congratulations on completing the introductory tutorial of **PINA** problems! There are several directions you can explore next: -# +# # 1. **Create Custom Problems**: Try building your own problems using the PINA framework, experiment with different PDEs, initial/boundary conditions, and data structures. -# +# # 2. **Explore the Problem Zoo**: Dive into the [`problem.zoo` module](https://mathlab.github.io/PINA/_rst/_code.html#problems-zoo) to find a variety of predefined problem setups and use them as a starting point or inspiration for your own. -# +# # 3. **...and many more!**: The possibilities are vast! Consider experimenting with different solver strategies, model architectures, or even implementing your own physical constraints. -# +# # For more examples and in-depth guides, be sure to check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial17/tutorial.py b/tutorials/tutorial17/tutorial.py index 2e28d19..629023d 100644 --- a/tutorials/tutorial17/tutorial.py +++ b/tutorials/tutorial17/tutorial.py @@ -2,79 +2,79 @@ # coding: utf-8 # # Tutorial: Introductory Tutorial: A Beginner’s Guide to PINA -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial17/tutorial.ipynb) -# +# #

# PINA Logo #

-# +# # Welcome to **PINA**! -# +# # PINA [1] is an open-source Python library designed for **Scientific Machine Learning (SciML)** tasks, particularly involving: -# +# # - **Physics-Informed Neural Networks (PINNs)** # - **Neural Operators (NOs)** # - **Reduced Order Models (ROMs)** # - **Graph Neural Networks (GNNs)** # - ... -# +# # Built on **PyTorch**, **PyTorch Lightning**, and **PyTorch Geometric**, it provides a **user-friendly, intuitive interface** for formulating and solving differential problems using neural networks. -# +# # This tutorial offers a **step-by-step guide** to using PINA—starting from basic to advanced techniques—enabling users to tackle a broad spectrum of differential problems with minimal code. -# -# -# +# +# +# -# ## The PINA Workflow -# +# ## The PINA Workflow +# #

# PINA Workflow #

-# +# # Solving a differential problem in **PINA** involves four main steps: -# +# # 1. ***Problem & Data*** -# Define the mathematical problem and its physical constraints using PINA’s base classes: +# Define the mathematical problem and its physical constraints using PINA’s base classes: # - `AbstractProblem` # - `SpatialProblem` -# - `InverseProblem` +# - `InverseProblem` # - ... -# +# # Then prepare inputs by discretizing the domain or importing numerical data. PINA provides essential tools like the `Conditions` class and the `pina.domain` module to facilitate domain sampling and ensure that the input data aligns with the problem's requirements. -# +# # > **👉 We have a dedicated [tutorial](https://mathlab.github.io/PINA/tutorial16/tutorial.html) to teach how to build a Problem from scratch — have a look if you're interested!** -# -# 2. ***Model Design*** +# +# 2. ***Model Design*** # Build neural network models as **PyTorch modules**. For graph-structured data, use **PyTorch Geometric** to build Graph Neural Networks. You can also import models from `pina.model` module! -# -# 3. ***Solver Selection*** +# +# 3. ***Solver Selection*** # Choose and configure a solver to optimize your model. Options include: # - **Supervised solvers**: `SupervisedSolver`, `ReducedOrderModelSolver` # - **Physics-informed solvers**: `PINN` and (many) variants -# - **Generative solvers**: `GAROM` +# - **Generative solvers**: `GAROM` # Solvers can be used out-of-the-box, extended, or fully customized. -# -# 4. ***Training*** +# +# 4. ***Training*** # Train your model using the `Trainer` class (built on **PyTorch Lightning**), which enables scalable and efficient training with advanced features. -# -# +# +# # By following these steps, PINA simplifies applying deep learning to scientific computing and differential problems. -# -# +# +# # ## A Simple Regression Problem in PINA # We'll start with a simple regression problem [2] of approximating the following function with a Neural Net model $\mathcal{M}_{\theta}$: -# $$y = x^3 + \epsilon, \quad \epsilon \sim \mathcal{N}(0, 9)$$ -# using only 20 samples: -# +# $$y = x^3 + \epsilon, \quad \epsilon \sim \mathcal{N}(0, 9)$$ +# using only 20 samples: +# # $$x_i \sim \mathcal{U}[-3, 3], \; \forall i \in \{1, \dots, 20\}$$ -# +# # Using PINA, we will: -# +# # - Generate a synthetic dataset. # - Implement a **Bayesian regressor**. # - Use **Monte Carlo (MC) Dropout** for **Bayesian inference** and **uncertainty estimation**. -# +# # This example highlights how PINA can be used for classic regression tasks with probabilistic modeling capabilities. Let's first import useful modules! # In[ ]: @@ -102,14 +102,14 @@ from pina.geometry import CartesianDomain # #### ***Problem & Data*** -# +# # We'll start by defining a `BayesianProblem` inheriting from `AbstractProblem` to handle input/output data. This is suitable when data is available. For other cases like PDEs without data, use: -# +# # - `SpatialProblem` – for spatial variables # - `TimeDependentProblem` – for temporal variables # - `ParametricProblem` – for parametric inputs # - `InverseProblem` – for parameter estimation from observations -# +# # but we will see this more in depth in a while! # In[21]: @@ -139,14 +139,14 @@ problem = BayesianProblem() # We highlight two very important features of PINA -# -# 1. **`LabelTensor` Structure** -# - Alongside the standard `torch.Tensor`, PINA introduces the `LabelTensor` structure, which allows **string-based indexing**. -# - Ideal for managing and stacking tensors with different labels (e.g., `"x"`, `"t"`, `"u"`) for improved clarity and organization. +# +# 1. **`LabelTensor` Structure** +# - Alongside the standard `torch.Tensor`, PINA introduces the `LabelTensor` structure, which allows **string-based indexing**. +# - Ideal for managing and stacking tensors with different labels (e.g., `"x"`, `"t"`, `"u"`) for improved clarity and organization. # - You can still use standard PyTorch tensors if needed. -# -# 2. **`Condition` Object** -# - The `Condition` object enforces the **constraints** that the model $\mathcal{M}_{\theta}$ must satisfy, such as boundary or initial conditions. +# +# 2. **`Condition` Object** +# - The `Condition` object enforces the **constraints** that the model $\mathcal{M}_{\theta}$ must satisfy, such as boundary or initial conditions. # - It ensures that the model adheres to the specific requirements of the problem, making constraint handling more intuitive and streamlined. # In[63]: @@ -167,14 +167,14 @@ print(f"Similarly to: \n {label_tensor[:, 0]=}") # #### ***Model Design*** -# -# We will now solve the problem using a **simple PyTorch Neural Network** with **Dropout**, which we will implement from scratch following [2]. +# +# We will now solve the problem using a **simple PyTorch Neural Network** with **Dropout**, which we will implement from scratch following [2]. # It's important to note that PINA provides a wide range of **state-of-the-art (SOTA)** architectures in the `pina.model` module, which you can explore further [here](https://mathlab.github.io/PINA/_rst/_code.html#models). -# +# # #### ***Solver Selection*** -# -# For this task, we will use a straightforward **supervised learning** approach by importing the `SupervisedSolver` from `pina.solvers`. The solver is responsible for defining the training strategy. -# +# +# For this task, we will use a straightforward **supervised learning** approach by importing the `SupervisedSolver` from `pina.solvers`. The solver is responsible for defining the training strategy. +# # The `SupervisedSolver` is designed to handle typical regression tasks effectively by minimizing the following loss function: # $$ # \mathcal{L}_{\rm{problem}} = \frac{1}{N}\sum_{i=1}^N @@ -184,14 +184,14 @@ print(f"Similarly to: \n {label_tensor[:, 0]=}") # $$ # \mathcal{L}(v) = \| v \|^2_2. # $$ -# +# # #### **Training** -# +# # Next, we will use the `Trainer` class to train the model. The `Trainer` class, based on **PyTorch Lightning**, offers many features that help: # - **Improve model accuracy** # - **Reduce training time and memory usage** -# - **Facilitate logging and visualization** -# +# - **Facilitate logging and visualization** +# # The great work done by the PyTorch Lightning team ensures a streamlined training process. # In[64]: @@ -230,15 +230,15 @@ trainer.train() # #### ***Model Training Complete! Now Visualize the Solutions*** -# +# # The model has been trained! Since we used **Dropout** during training, the model is probabilistic (Bayesian) [3]. This means that each time we evaluate the forward pass on the input points $x_i$, the results will differ due to the stochastic nature of Dropout. -# +# # To visualize the model's predictions and uncertainty, we will: -# +# # 1. **Evaluate the Forward Pass**: Perform multiple forward passes to get different predictions for each input $x_i$. # 2. **Compute the Mean**: Calculate the average prediction $\mu_\theta$ across all forward passes. # 3. **Compute the Standard Deviation**: Calculate the variability of the predictions $\sigma_\theta$, which indicates the model's uncertainty. -# +# # This allows us to understand not only the predicted values but also the confidence in those predictions. # In[65]: @@ -266,32 +266,32 @@ plt.show() # ## PINA for Physics-Informed Machine Learning -# +# # In the previous section, we used PINA for **supervised learning**. However, one of its main strengths lies in **Physics-Informed Machine Learning (PIML)**, specifically through **Physics-Informed Neural Networks (PINNs)**. -# +# # ### What Are PINNs? -# +# # PINNs are deep learning models that integrate the laws of physics directly into the training process. By incorporating **differential equations** and **boundary conditions** into the loss function, PINNs allow the modeling of complex physical systems while ensuring the predictions remain consistent with scientific laws. -# +# # ### Solving a 2D Poisson Problem -# +# # In this section, we will solve a **2D Poisson problem** with **Dirichlet boundary conditions** on an **hourglass-shaped domain** using a simple PINN [4]. You can explore other PINN variants, e.g. [5] or [6] in PINA by visiting the [PINA solvers documentation](https://mathlab.github.io/PINA/_rst/_code.html#solvers). We aim to solve the following 2D Poisson problem: -# +# # $$ # \begin{cases} # \Delta u(x, y) = \sin{(\pi x)} \sin{(\pi y)} & \text{in } D, \\ -# u(x, y) = 0 & \text{on } \partial D +# u(x, y) = 0 & \text{on } \partial D # \end{cases} # $$ -# +# # where $D$ is an **hourglass-shaped domain** defined as the difference between a **Cartesian domain** and two intersecting **ellipsoids**, and $\partial D$ is the boundary of the domain. -# +# # ### Building Complex Domains -# +# # PINA allows you to build complex geometries easily. It provides many built-in domain shapes and Boolean operators for combining them. For this problem, we will define the hourglass-shaped domain using the existing `CartesianDomain` and `EllipsoidDomain` classes, with Boolean operators like `Difference` and `Union`. -# +# # > **👉 If you are interested in exploring the `domain` module in more detail, check out [this tutorial](https://mathlab.github.io/PINA/_rst/tutorials/tutorial6/tutorial.html).** -# +# # In[66]: @@ -332,7 +332,7 @@ border_samples = border.sample(n=1000, mode="random") # #### Plotting the domain -# +# # Nice! Now that we have built the domain, let's try to plot it # In[67]: @@ -359,11 +359,11 @@ plt.show() # #### Writing the Poisson Problem Class -# -# Very good! Now we will implement the problem class for the 2D Poisson problem. Unlike the previous examples, where we inherited from `AbstractProblem`, for this problem, we will inherit from the `SpatialProblem` class. -# +# +# Very good! Now we will implement the problem class for the 2D Poisson problem. Unlike the previous examples, where we inherited from `AbstractProblem`, for this problem, we will inherit from the `SpatialProblem` class. +# # The reason for this is that the Poisson problem involves **spatial variables** as input, so we use `SpatialProblem` to handle such cases. -# +# # This will allow us to define the problem with spatial dependencies and set up the neural network model accordingly. # In[69]: @@ -401,12 +401,12 @@ poisson_problem = Poisson() # As you can see, writing the problem class for a differential equation in PINA is straightforward! The main differences are: -# +# # - We inherit from **`SpatialProblem`** instead of `AbstractProblem` to account for spatial variables. # - We use **`domain`** and **`equation`** inside the `Condition` to define the problem. -# +# # The `Equation` class can be very useful for creating modular problem classes. If you're interested, check out [this tutorial](https://mathlab.github.io/PINA/_rst/tutorial12/tutorial.html) for more details. There's also a dedicated [tutorial](https://mathlab.github.io/PINA/_rst/tutorial16/tutorial.html) for building custom problems! -# +# # Once the problem class is set, we need to **sample the domain** to obtain the data. PINA will automatically handle this, and if you forget to sample, an error will be raised before training begins 😉. # In[70]: @@ -421,13 +421,13 @@ print(f" {poisson_problem.are_all_domains_discretised=}") # ### Building the Model -# +# # After setting the problem and sampling the domain, the next step is to **build the model** $\mathcal{M}_{\theta}$. -# +# # For this, we will use the custom PINA models available [here](https://mathlab.github.io/PINA/_rst/_code.html#models). Specifically, we will use a **feed-forward neural network** by importing the `FeedForward` class. -# -# This neural network takes the **coordinates** (in this case `['x', 'y']`) as input and outputs the unknown field of the Poisson problem. -# +# +# This neural network takes the **coordinates** (in this case `['x', 'y']`) as input and outputs the unknown field of the Poisson problem. +# # In this tutorial, the neural network is composed of 2 hidden layers, each with 120 neurons and tanh activation. # In[72]: @@ -444,30 +444,30 @@ model = FeedForward( # ### Solver Selection -# +# # The thir part of the PINA pipeline involves using a **Solver**. -# +# # In this tutorial, we will use the **classical PINN** solver. However, many other variants are also available and we invite to try them! -# +# # #### Loss Function in PINA -# +# # The loss function in the **classical PINN** is defined as follows: -# +# # $$\theta_{\rm{best}}=\min_{\theta}\mathcal{L}_{\rm{problem}}(\theta), \quad \mathcal{L}_{\rm{problem}}(\theta)= \frac{1}{N_{D}}\sum_{i=1}^N # \mathcal{L}(\Delta\mathcal{M}_{\theta}(\mathbf{x}_i, \mathbf{y}_i) - \sin(\pi x_i)\sin(\pi y_i)) + # \frac{1}{N}\sum_{i=1}^N # \mathcal{L}(\mathcal{M}_{\theta}(\mathbf{x}_i, \mathbf{y}_i))$$ -# +# # This loss consists of: # 1. The **differential equation residual**: Ensures the model satisfies the Poisson equation. # 2. The **boundary condition**: Ensures the model satisfies the Dirichlet boundary condition. -# +# # ### Training -# +# # For the last part of the pipeline we need a `Trainer`. We will train the model for **1000 epochs** using the default optimizer parameters. These parameters can be adjusted as needed. For more details, check the solvers documentation [here](https://mathlab.github.io/PINA/_rst/_code.html#solvers). -# +# # To track metrics during training, we use the **`MetricTracker`** class. -# +# # > **👉 Want to know more about `Trainer` and how to boost PINA performance, check out [this tutorial](https://mathlab.github.io/PINA/_rst/tutorials/tutorial11/tutorial.html).** # In[73]: @@ -526,28 +526,28 @@ with torch.no_grad(): # ## What's Next? -# +# # Congratulations on completing the introductory tutorial of **PINA**! Now that you have a solid foundation, here are a few directions you can explore: -# +# # 1. **Explore Advanced Solvers**: Dive into more advanced solvers like **SAPINN** or **RBAPINN** and experiment with different variations of Physics-Informed Neural Networks. # 2. **Apply PINA to New Problems**: Try solving other types of differential equations or explore inverse problems and parametric problems using the PINA framework. # 3. **Optimize Model Performance**: Use the `Trainer` class to enhance model performance by exploring features like dynamic learning rates, early stopping, and model checkpoints. -# +# # 4. **...and many more!** — There are countless directions to further explore, from testing on different problems to refining the model architecture! -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). -# -# +# +# # ### References -# +# # [1] *Coscia, Dario, et al. "Physics-informed neural networks for advanced modeling." Journal of Open Source Software, 2023.* -# +# # [2] *Hernández-Lobato, José Miguel, and Ryan Adams. "Probabilistic backpropagation for scalable learning of bayesian neural networks." International conference on machine learning, 2015.* -# +# # [3] *Gal, Yarin, and Zoubin Ghahramani. "Dropout as a bayesian approximation: Representing model uncertainty in deep learning." International conference on machine learning, 2016.* -# +# # [4] *Raissi, Maziar, Paris Perdikaris, and George E. Karniadakis. "Physics-informed neural networks: A deep learning framework for solving forward and inverse problems involving nonlinear partial differential equations." Journal of Computational Physics, 2019.* -# +# # [5] *McClenny, Levi D., and Ulisses M. Braga-Neto. "Self-adaptive physics-informed neural networks." Journal of Computational Physics, 2023.* -# +# # [6] *Anagnostopoulos, Sokratis J., et al. "Residual-based attention in physics-informed neural networks." Computer Methods in Applied Mechanics and Engineering, 2024.* diff --git a/tutorials/tutorial18/tutorial.py b/tutorials/tutorial18/tutorial.py index c0edad6..24a0f81 100644 --- a/tutorials/tutorial18/tutorial.py +++ b/tutorials/tutorial18/tutorial.py @@ -2,29 +2,29 @@ # coding: utf-8 # # Tutorial: Introduction to Solver classes -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial18/tutorial.ipynb) -# +# # In this tutorial, we will explore the Solver classes in PINA, that are the core components for optimizing models. Solvers are designed to manage and execute the optimization process, providing the flexibility to work with various types of neural networks and loss functions. We will show how to use this class to select and implement different solvers, such as Supervised Learning, Physics-Informed Neural Networks (PINNs), and Generative Learning solvers. By the end of this tutorial, you'll be equipped to easily choose and customize solvers for your own tasks, streamlining the model training process. -# +# # ## Introduction to Solvers -# +# # [`Solvers`](https://mathlab.github.io/PINA/_rst/_code.html#solvers) are versatile objects in PINA designed to manage the training and optimization of machine learning models. They handle key components of the learning process, including: -# -# - Loss function minimization +# +# - Loss function minimization # - Model optimization (optimizer, schedulers) # - Validation and testing workflows -# +# # PINA solvers are built on top of the [PyTorch Lightning `LightningModule`](https://lightning.ai/docs/pytorch/stable/common/lightning_module.html), which provides a structured and scalable training framework. This allows solvers to leverage advanced features such as distributed training, early stopping, and logging — all with minimal setup. -# +# # ## Solvers Hierarchy: Single and MultiSolver -# +# # PINA provides two main abstract interfaces for solvers, depending on whether the training involves a single model or multiple models. These interfaces define the base functionality that all specific solver implementations inherit from. -# +# # ### 1. [`SingleSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/solver_interface.html) -# +# # This is the abstract base class for solvers that train **a single model**, such as in standard supervised learning or physics-informed training. All specific solvers (e.g., `SupervisedSolver`, `PINN`) inherit from this interface. -# +# # **Arguments:** # - `problem` – The problem to be solved. # - `model` – The neural network model. @@ -32,13 +32,13 @@ # - `scheduler` – Defaults to `torch.optim.lr_scheduler.ConstantLR`. # - `weighting` – Optional loss weighting schema., see [here](https://mathlab.github.io/PINA/_rst/_code.html#losses-and-weightings). We weight already for you! # - `use_lt` – Whether to use LabelTensors as input. -# +# # --- -# +# # ### 2. [`MultiSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/multi_solver_interface.html) -# +# # This is the abstract base class for solvers involving **multiple models**, such as in GAN architectures or ensemble training strategies. All multi-model solvers (e.g., `DeepEnsemblePINN`, `GAROM`) inherit from this interface. -# +# # **Arguments:** # - `problem` – The problem to be solved. # - `models` – The model or models used for training. @@ -46,19 +46,19 @@ # - `schedulers` – Defaults to `torch.optim.lr_scheduler.ConstantLR`. # - `weightings` – Optional loss weighting schema, see [here](https://mathlab.github.io/PINA/_rst/_code.html#losses-and-weightings). We weight already for you! # - `use_lt` – Whether to use LabelTensors as input. -# +# # --- -# -# These base classes define the structure and behavior of solvers in PINA, allowing you to create customized training strategies while leveraging PyTorch Lightning's features under the hood. -# +# +# These base classes define the structure and behavior of solvers in PINA, allowing you to create customized training strategies while leveraging PyTorch Lightning's features under the hood. +# # These classes are used to define the backbone, i.e. setting the problem, the model(s), the optimizer(s) and scheduler(s), but miss a key component the `optimization_cycle` method. -# -# +# +# # ## Optimization Cycle # The `optimization_cycle` method is the core function responsible for computing losses for **all conditions** in a given training batch. Each condition (e.g. initial condition, boundary condition, PDE residual) contributes its own loss, which is tracked and returned in a dictionary. This method should return a dictionary mapping **condition names** to their respective **scalar loss values**. -# +# # For supervised learning tasks, where each condition consists of an input-target pair, for example, the `optimization_cycle` may look like this: -# +# # ```python # def optimization_cycle(self, batch): # """ @@ -73,12 +73,12 @@ # return condition_loss # ``` # In PINA, a **batch** is structured as a list of tuples, where each tuple corresponds to a specific training condition. Each tuple contains: -# +# # - The **name of the condition** # - A **dictionary of data** associated with that condition -# +# # for example: -# +# # ```python # batch = [ # ("condition1", {"input": ..., "target": ...}), @@ -86,52 +86,52 @@ # ("condition3", {"input": ..., "target": ...}), # ] # ``` -# +# # Fortunately, you don't need to implement the `optimization_cycle` yourself in most cases — PINA already provides default implementations tailored to common solver types. These implementations are available through the solver interfaces and cover various training strategies. -# -# 1. [`PINNInterface`](https://mathlab.github.io/PINA/_rst/solver/physics_informed_solver/pinn_interface.html) -# Implements the optimization cycle for **physics-based solvers** (e.g., PDE residual minimization) as well as other useful methods to compute PDE residuals. +# +# 1. [`PINNInterface`](https://mathlab.github.io/PINA/_rst/solver/physics_informed_solver/pinn_interface.html) +# Implements the optimization cycle for **physics-based solvers** (e.g., PDE residual minimization) as well as other useful methods to compute PDE residuals. # ➤ [View method](https://mathlab.github.io/PINA/_rst/solver/physics_informed_solver/pinn_interface.html#pina.solver.physics_informed_solver.pinn_interface.PINNInterface.optimization_cycle) -# -# 2. [`SupervisedSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html) -# Defines the optimization cycle for **supervised learning tasks**, including traditional regression and classification. +# +# 2. [`SupervisedSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html) +# Defines the optimization cycle for **supervised learning tasks**, including traditional regression and classification. # ➤ [View method](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html#pina.solver.supervised_solver.supervised_solver_interface.SupervisedSolverInterface.optimization_cycle) -# -# 3. [`DeepEnsembleSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/ensemble_solver/ensemble_solver_interface.html) -# Provides the optimization logic for **deep ensemble methods**, commonly used for uncertainty quantification or robustness. +# +# 3. [`DeepEnsembleSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/ensemble_solver/ensemble_solver_interface.html) +# Provides the optimization logic for **deep ensemble methods**, commonly used for uncertainty quantification or robustness. # ➤ [View method](https://mathlab.github.io/PINA/_rst/solver/ensemble_solver/ensemble_solver_interface.html#pina.solver.ensemble_solver.ensemble_solver_interface.DeepEnsembleSolverInterface.optimization_cycle) -# +# # These ready-to-use implementations ensure that your solvers are properly structured and compatible with PINA’s training workflow. You can also inherit and override them to fit more specialized needs. They only require, the following arguments: # **Arguments:** # - `problem` – The problem to be solved. # - `loss` - The loss to be minimized # - `weightings` – Optional loss weighting schema. # - `use_lt` – Whether to use LabelTensors as input. -# +# # ## Structure a Solver with Multiple Inheritance: -# +# # Thanks to PINA’s modular design, creating a custom solver is straightforward using **multiple inheritance**. You can combine different interfaces to define both the **optimization logic** and the **model structure**. -# +# # - **`PINN` Solver** -# - Inherits from: -# - [`PINNInterface`](https://mathlab.github.io/PINA/_rst/solver/physics_informed_solver/pinn_interface.html) → physics-based optimization loop +# - Inherits from: +# - [`PINNInterface`](https://mathlab.github.io/PINA/_rst/solver/physics_informed_solver/pinn_interface.html) → physics-based optimization loop # - [`SingleSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/solver_interface.html) → training a single model -# +# # - **`SupervisedSolver`** -# - Inherits from: -# - [`SupervisedSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html) → data-driven optimization loop +# - Inherits from: +# - [`SupervisedSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html) → data-driven optimization loop # - [`SingleSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/solver_interface.html) → training a single model -# +# # - **`GAROM`** (a variant of GAN) -# - Inherits from: -# - [`SupervisedSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html) → data-driven optimization loop +# - Inherits from: +# - [`SupervisedSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/supervised_solver/supervised_solver_interface.html) → data-driven optimization loop # - [`MultiSolverInterface`](https://mathlab.github.io/PINA/_rst/solver/multi_solver_interface.html) → training multiple models (e.g., generator and discriminator) -# +# # This structure promotes **code reuse** and **extensibility**, allowing you to quickly prototype new solver strategies by reusing core training and optimization logic. -# +# # ## Let's try to build some solvers! -# -# We will now start building a simple supervised solver in PINA. Let's first import useful modules! +# +# We will now start building a simple supervised solver in PINA. Let's first import useful modules! # In[ ]: @@ -159,7 +159,7 @@ from pina.problem.zoo import SupervisedProblem # Since we are using only one model for this task, we will inherit from two base classes: -# +# # - `SingleSolverInterface`: This ensures we are working with a single model. # - `SupervisedSolverInterface`: This allows us to use supervised learning strategies for training the model. @@ -189,9 +189,9 @@ class MyFirstSolver(SupervisedSolverInterface, SingleSolverInterface): # By default, Python follows a specific method resolution order (MRO) when a class inherits from multiple parent classes. This means that the initialization (`__init__`) method is called based on the order of inheritance. -# -# Since we inherit from `SupervisedSolverInterface` first, Python will call the `__init__` method from `SupervisedSolverInterface` (initialize `problem`, `loss`, `weighting` and `use_lt`) before calling the `__init__` method from `SingleSolverInterface` (initialize `model`, `optimizer`, `scheduler`). This allows us to customize the initialization process for our custom solver. -# +# +# Since we inherit from `SupervisedSolverInterface` first, Python will call the `__init__` method from `SupervisedSolverInterface` (initialize `problem`, `loss`, `weighting` and `use_lt`) before calling the `__init__` method from `SingleSolverInterface` (initialize `model`, `optimizer`, `scheduler`). This allows us to customize the initialization process for our custom solver. +# # We will learn a very simple problem, try to learn $y=\sin(x)$. # In[51]: @@ -207,23 +207,23 @@ model = FeedForward(1, 1) # If we now try to initialize the solver `MyFirstSolver` we will get the following error: -# +# # ```python # --------------------------------------------------------------------------- # TypeError Traceback (most recent call last) # Cell In[41], line 1 # ----> 1 MyFirstSolver(problem, model) -# +# # TypeError: Can't instantiate abstract class MyFirstSolver with abstract method loss_data # ``` -# +# # ### Data and Physics Loss # The error above is because in PINA, all solvers must specify how to compute the loss during training. There are two main types of losses that can be computed, depending on the nature of the problem: -# +# # 1. **`loss_data`**: Computes the **data loss** between the model's output and the true solution. This is typically used in **supervised learning** setups, where we have ground truth data to compare the model's predictions. It expects some `input` (tensor, graph, ...) and a `target` (tensor, graph, ...) -# +# # 2. **`loss_phys`**: Computes the **physics loss** for **physics-informed solvers** (PINNs). This loss is based on the residuals of the governing equations that model physical systems, enforcing the equations during training. It expects some `samples` (`LabelTensor`) and an `equation` (`Equation`) -# +# # Therefore our implementation becomes: # In[52]: @@ -268,33 +268,33 @@ _ = trainer.test() # ## A Summary on Solvers -# +# # Solvers in PINA play a critical role in training and optimizing machine learning models, especially when working with complex problems like physics-informed neural networks (PINNs) or standard supervised learning. Here’s a quick recap of the key concepts we've covered: -# +# # 1. **Solver Interfaces**: # - **`SingleSolverInterface`**: For solvers using one model (e.g., a standard supervised solver or a single physics-informed model). # - **`MultiSolverInterface`**: For solvers using multiple models (e.g., Generative Adversarial Networks (GANs)). -# +# # 2. **Loss Functions**: # - **`loss_data`**: Computes the loss for supervised solvers, typically comparing the model's predictions to the true targets. # - **`loss_phys`**: Computes the physics loss for PINNs, typically using the residuals of a physical equation to enforce consistency with the physics of the system. -# +# # 3. **Custom Solver Implementation**: # - You can create custom solvers by inheriting from base classes such as `SingleSolverInterface`. The **`optimization_cycle`** method must be implemented to define how to compute the loss for each batch. # - `SupervisedSolverInterface`, `PINNInterface` already implement the `optimization_cycle` for you! -# -# +# +# # By understanding and implementing solvers in PINA, you can build flexible, scalable models that can be optimized both with traditional supervised learning techniques and more specialized, physics-based methods. # ## What's Next? -# +# # Congratulations on completing the tutorial on solver classes! Now that you have a solid foundation, here are a few directions you can explore: -# -# +# +# # 1. **Physics Solvers**: Try to implement your own physics-based solver. Can you do it? This will involve creating a custom loss function that enforces the physics of a given problem insied `loss_phys`. -# +# # 2. **Multi-Model Solvers**: Take it to the next level by exploring multi-model solvers, such as GANs or ensemble-based solvers. You could implement and train models that combine the strengths of multiple neural networks. -# +# # 3. **...and many more!**: There are countless directions to further explore, try to look at our `solver` for example! -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial19/tutorial.py b/tutorials/tutorial19/tutorial.py index 8f576aa..c5af084 100644 --- a/tutorials/tutorial19/tutorial.py +++ b/tutorials/tutorial19/tutorial.py @@ -2,13 +2,13 @@ # coding: utf-8 # # Tutorial: Data structure for SciML: `Tensor`, `LabelTensor`, `Data` and `Graph` -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial19/tutorial.ipynb) -# +# # In this tutorial, we’ll quickly go through the basics of Data Structures for Scientific Machine Learning, convering: # 1. **PyTorch Tensors** / **PINA LabelTensors** # 2. **PyTorch Geometric Data** / **PINA Graph** -# +# # first let's import the data structures we will use! # In[ ]: @@ -34,9 +34,9 @@ from pina import LabelTensor, Graph # ## PyTorch Tensors -# +# # A **tensor** is a multi-dimensional matrix used for storing and manipulating data in PyTorch. It's the basic building block for all computations in PyTorch, including deep learning models. -# +# # You can create a tensor in several ways: # In[2]: @@ -89,13 +89,13 @@ if torch.cuda.is_available(): # To know more about PyTorch Tensors, see the dedicated tutorial done by the PyTorch team [here](https://pytorch.org/tutorials/beginner/introyt/tensors_deeper_tutorial.html). # ## Label Tensors -# +# # In scientific machine learning, especially when working with **Physics-Informed Neural Networks (PINNs)**, handling tensors effectively is crucial. Often, we deal with many indices that represent physical quantities such as spatial and temporal coordinates, making it vital to ensure we use the correct indexing. -# +# # For instance, in PINNs, if the wrong index is used to represent the coordinates of a physical domain, it could lead to incorrect calculations of derivatives, integrals, or residuals. This can significantly affect the accuracy and correctness of the model. -# +# # ### What are Label Tensors? -# +# # **Label Tensors** are a specialized type of tensor used to keep track of indices that represent specific labels. Similar to torch tensor we can perform operation, but the slicing is simplified by using indeces: # In[7]: @@ -109,12 +109,12 @@ label_tensor = LabelTensor(tensor, labels=["x", "y"]) # The label tensor is initialized by passing the tensor, and a set of labels. Specifically, the labels must match the following conditions: -# +# # - At each dimension, the number of labels must match the size of the dimension. # - At each dimension, the labels must be unique. -# +# # For example: -# +# # In[9]: @@ -137,9 +137,9 @@ print(f"{tensor.full_labels=}") # ### Label Tensors slicing -# -# One of the powerful features of label tensors is the ability to easily slice and extract specific parts of the tensor based on labels, just like regular PyTorch tensors but with the ease of labels. -# +# +# One of the powerful features of label tensors is the ability to easily slice and extract specific parts of the tensor based on labels, just like regular PyTorch tensors but with the ease of labels. +# # Here’s how slicing works with label tensors. Suppose we have a label tensor that contains both spatial and temporal data, and we want to slice specific parts of this data to focus on certain time intervals or spatial regions. # In[26]: @@ -181,16 +181,16 @@ print(f"Similar to: {label_tensor[slice(0, 4, 2), :]=}") # ## PyTorch Geometric Data # PyTorch Geometric (PyG) extends PyTorch to handle graph-structured data. It provides utilities to represent graphs and perform graph-based learning tasks such as node classification, graph classification, and more. -# +# # ### Graph Data Structure # PyTorch Geometric uses a custom `Data` object to store graph data. The `Data` object contains the following attributes: -# +# # - **x**: Node features (tensor of shape `[num_nodes, num_features]`) -# +# # - **edge_index**: Edge indices (tensor of shape `[2, num_edges]`), representing the graph's connectivity -# +# # - **edge_attr**: Edge features (optional, tensor of shape `[num_edges, num_edge_features]`) -# +# # - **y**: Target labels for nodes/graphs (optional) # In[32]: @@ -229,7 +229,7 @@ print(out) # Output node features after applying GCN # ## PINA Graph -# +# # If you've understood Label Tensors and Data in PINA, then you're well on your way to grasping how **PINA Graph** works. Simply put, a **Graph** in PINA is a `Data` object with extra methods for handling label tensors. We highly suggest to use `Graph` instead of `Data` in PINA, expecially when using label tensors. # In[36]: @@ -296,13 +296,13 @@ input_[0] # ## What's Next? -# +# # Congratulations on completing the tutorials on the **PINA Data Structures**! You now have a solid foundation in using the different data structures within PINA, such as **Tensors**, **Label Tensors**, and **Graphs**. Here are some exciting next steps you can take to continue your learning journey: -# +# # 1. **Deep Dive into Label Tensors**: Check the documentation of [`LabelTensor`](https://mathlab.github.io/PINA/_rst/label_tensor.html) to learn more about the available methods. -# +# # 2. **Working with Graphs in PINA**: In PINA we implement many graph structures, e.g. `KNNGraph`, `RadiusGraph`, .... see [here](https://mathlab.github.io/PINA/_rst/_code.html#graphs-structures) for further details. -# +# # 3. **...and many more!**: Consider exploring `LabelTensor` for PINNs! -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial2/tutorial.py b/tutorials/tutorial2/tutorial.py index fb4d812..6136a6e 100644 --- a/tutorials/tutorial2/tutorial.py +++ b/tutorials/tutorial2/tutorial.py @@ -2,11 +2,11 @@ # coding: utf-8 # # Tutorial: Enhancing PINNs with Extra Features to solve the Poisson Problem -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial2/tutorial.ipynb) -# +# # This tutorial presents how to solve with Physics-Informed Neural Networks (PINNs) a 2D Poisson problem with Dirichlet boundary conditions. We will train with standard PINN's training, and with extrafeatures. For more insights on extrafeature learning please read [*An extended physics informed neural network for preliminary analysis of parametric optimal control problems*](https://www.sciencedirect.com/science/article/abs/pii/S0898122123002018). -# +# # First of all, some useful imports. # In[ ]: @@ -44,10 +44,10 @@ warnings.filterwarnings("ignore") # \end{cases} # \end{equation} # where $D$ is a square domain $[0,1]^2$, and $\Gamma_i$, with $i=1,...,4$, are the boundaries of the square. -# +# # The Poisson problem is written in **PINA** code as a class. The equations are written as *conditions* that should be satisfied in the corresponding domains. The *solution* # is the exact solution which will be compared with the predicted one. If interested in how to write problems see [this tutorial](https://mathlab.github.io/PINA/_rst/tutorials/tutorial16/tutorial.html). -# +# # We will directly import the problem from `pina.problem.zoo`, which contains a vast list of PINN problems and more. # In[4]: @@ -76,7 +76,7 @@ problem.discretise_domain( # ## Solving the problem with standard PINNs # After the problem, the feed-forward neural network is defined, through the class `FeedForward`. This neural network takes as input the coordinates (in this case $x$ and $y$) and provides the unkwown field of the Poisson problem. The residual of the equations are evaluated at several sampling points and the loss minimized by the neural network is the sum of the residuals. -# +# # In this tutorial, the neural network is composed by two hidden layers of 10 neurons each, and it is trained for 1000 epochs with a learning rate of 0.006 and $l_2$ weight regularization set to $10^{-8}$. These parameters can be modified as desired. We set the `train_size` to 0.8 and `test_size` to 0.2, this mean that the discretised points will be divided in a 80%-20% fashion, where 80% will be used for training and the remaining 20% for testing. # In[5]: @@ -112,7 +112,7 @@ trainer_base.train() # Now we plot the results using `matplotlib`. -# The solution predicted by the neural network is plotted on the left, the exact one is in the center and on the right the error between the exact and the predicted solutions is showed. +# The solution predicted by the neural network is plotted on the left, the exact one is in the center and on the right the error between the exact and the predicted solutions is showed. # In[6]: @@ -157,17 +157,17 @@ plot_solution(solver=pinn) # ## Solving the problem with extra-features PINNs # Now, the same problem is solved in a different way. -# A new neural network is now defined, with an additional input variable, named extra-feature, which coincides with the forcing term in the Laplace equation. +# A new neural network is now defined, with an additional input variable, named extra-feature, which coincides with the forcing term in the Laplace equation. # The set of input variables to the neural network is: -# +# # \begin{equation} # [x, y, k(x, y)], \text{ with } k(x, y)= 2\pi^2\sin{(\pi x)}\sin{(\pi y)}, # \end{equation} -# +# # where $x$ and $y$ are the spatial coordinates and $k(x, y)$ is the added feature which is equal to the forcing term. -# +# # This feature is initialized in the class `SinSin`, which is a simple `torch.nn.Module`. After declaring such feature, we can just adjust the `FeedForward` class by creating a subclass `FeedForwardWithExtraFeatures` with an adjusted forward method and the additional attribute `extra_features`. -# +# # Finally, we perform the same training as before: the problem is `Poisson`, the network is composed by the same number of neurons and optimizer parameters are equal to previous test, the only change is the new extra feature. # In[8]: @@ -236,14 +236,14 @@ plot_solution(solver=pinn_feat) # ## Solving the problem with learnable extra-features PINNs # We can still do better! -# +# # Another way to exploit the extra features is the addition of learnable parameter inside them. # In this way, the added parameters are learned during the training phase of the neural network. In this case, we use: -# +# # \begin{equation} # k(x, \mathbf{y}) = \beta \sin{(\alpha x)} \sin{(\alpha y)}, # \end{equation} -# +# # where $\alpha$ and $\beta$ are the abovementioned parameters. # Their implementation is quite trivial: by using the class `torch.nn.Parameter` we cam define all the learnable parameters we need, and they are managed by `autograd` module! @@ -348,15 +348,15 @@ _ = trainer_learn.test() # ## What's Next? -# +# # Congratulations on completing the two-dimensional Poisson tutorial of **PINA**! Now that you've learned the basics, there are multiple directions you can explore: -# +# # 1. **Train the Network for Longer**: Continue training the network for a longer duration or experiment with different layer sizes to assess the final accuracy. -# +# # 2. **Propose New Types of Extrafeatures**: Experiment with new extrafeatures and investigate how they affect the learning process. -# +# # 3. **Leverage Extrafeature Training for Complex Problems**: Apply extrafeature training techniques to more complex problems to improve model performance. -# +# # 4. **... and many more!.**: There are endless possibilities! Continue exploring and experimenting with new ideas. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial20/tutorial.py b/tutorials/tutorial20/tutorial.py index 0bcd4e9..692a255 100644 --- a/tutorials/tutorial20/tutorial.py +++ b/tutorials/tutorial20/tutorial.py @@ -2,15 +2,15 @@ # coding: utf-8 # # Tutorial: Introductory Tutorial: Supervised Learning with PINA -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial20/tutorial.ipynb) -# -# +# +# # > ##### ⚠️ ***Before starting:*** # > We assume you are already familiar with the concepts covered in the [Getting started with PINA](https://mathlab.github.io/PINA/_tutorial.html#getting-started-with-pina) tutorials. If not, we strongly recommend reviewing them before exploring this advanced topic. -# +# # In this tutorial, we will demonstrate a typical use case of **PINA** for Supervised Learning training. We will cover the basics of training a Supervised Solver with PINA, if you want to go further into PINNs look at our dedicated [tutorials](https://mathlab.github.io/PINA/_tutorial.html#supervised-learning) on the topic. -# +# # Let's start by importing the useful modules: # In[1]: @@ -42,22 +42,22 @@ from pina.problem.zoo import SupervisedProblem # ## Building a Neural Implicit Field for a Sphere -# +# # In this tutorial, we will construct a **Neural Implicit Field** to learn the **Signed Distance Function (SDF)** of a sphere. The problem is relatively simple: we aim to learn a function $d_\theta$, parameterized by a neural network, that captures the signed distance to the surface of a sphere. -# +# # The function $d_\theta(\mathbf{x})$$ should satisfy the following properties: -# -# - $d_\theta(\mathbf{x}) = 0$ on the surface of the sphere -# - $d_\theta(\mathbf{x}) > 0$ outside the sphere -# - $d_\theta(\mathbf{x}) < 0$ inside the sphere -# +# +# - $d_\theta(\mathbf{x}) = 0$ on the surface of the sphere +# - $d_\theta(\mathbf{x}) > 0$ outside the sphere +# - $d_\theta(\mathbf{x}) < 0$ inside the sphere +# # This setup allows us to implicitly represent the geometry of the sphere through the learned function. -# +# # ### Mathematical Description -# +# # We define the signed distance function (SDF) for a sphere centered at the origin with radius $r$ as: # $d(\mathbf{x}) = \|\mathbf{x}\| - r$, where $\mathbf{x} \in \mathbb{R}^3$ is a point in 3D space. -# +# # Our goal is to approximate this function using a neural network: $d_\theta(\mathbf{x}) \approx d(\mathbf{x})$ with a Neural Network. Let's start by generating the data for the problem by: # 1. Sample random 3D points within a bounding cube (e.g., $[-1.5, 1.5]^3$). # 2. Compute their ground truth signed distances from a sphere of radius $r$ centered at the origin. @@ -84,7 +84,7 @@ def generate_sdf_data(num_points=1000000, radius=1.0, cube_bound=1.5): # ### Visualizing the Data -# +# # To better understand the problem and the nature of the solutions, we can visualize the generated data: # In[3]: @@ -113,9 +113,9 @@ plt.show() # ## Creating the Problem -# +# # The problem we will define is a basic `SupervisedProblem`, where the inputs are the coordinates and the outputs are the corresponding Signed Distance Function (SDF) values. -# +# # > **👉 We have a dedicated [tutorial](https://mathlab.github.io/PINA/tutorial16/tutorial.html) to teach how to build a Problem from scratch — have a look if you're interested!** # In[4]: @@ -125,16 +125,16 @@ problem = SupervisedProblem(coords, sdf) # ## Solving the Problem with Supervised Solver -# +# # We will use the `SupervisedSolver` to solve the task. A Supervised Solver in PINA aims to find a mapping between an input \( x \) and an output \( y \). # Given a PINA `model` $\mathcal{M}$, the following loss function is minimized during training: -# +# # $$ # \mathcal{L}_{\rm{supervised}} = \frac{1}{N}\sum_{i=1}^N \mathcal{l}(y_i, \mathcal{M}(x_i)), # $$ -# +# # where $l$ is a specific loss function, typically the MSE (Mean Squared Error). -# +# # ### Specify the Loss Function # By default, the loss function applies a forward pass of the `model` on the input and compares it to the target using the `loss` attribute of `SupervisedSolver`. The [`loss_data`](https://mathlab.github.io/PINA/_rst/solver/supervised.html#pina.solver.supervised.SupervisedSolver.loss_data) function computes the loss for supervised solvers, and it can be overridden by the user to match specific needs (e.g., performing pre-process operations on the input, post-process operations on the output, etc.). @@ -162,9 +162,9 @@ _ = trainer.test() # ## Visualizing the Predictions -# +# # As we can see, we have achieved a very low MSE, even after training for only one epoch. Now, we will visualize the results in the same way as we did previously: -# +# # We will plot the predicted Signed Distance Function (SDF) values alongside the true SDF values to evaluate the model's performance. # In[6]: @@ -222,9 +222,9 @@ plt.show() # Nice! We can see that the network is correctly learning the signed distance function! Let's now visualize the rendering of the sphere surface learned by the network. -# +# # ### Visualizing the Sphere Surface -# +# # To visualize the surface, we will extract the level set where the SDF equals zero and plot the resulting sphere. This will show how well the network has learned the geometry of the object. # In[7]: @@ -262,14 +262,14 @@ plt.show() # ## What's Next? -# +# # Congratulations on completing the introductiory tutorial on supervised solver! Now that you have a solid foundation, here are a few directions you can explore: -# -# +# +# # 1. **Experiment with Training Duration & Network Architecture**: Try different training durations and tweak the network architecture to optimize performance. -# +# # 2. **Explore Other Models in `pina.model`**: Check out other models available in `pina.model` or design your own custom PyTorch module to suit your needs. -# +# # 3. **... and many more!**: The possibilities are vast! Continue experimenting with advanced configurations, solvers, and other features in PINA. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial21/tutorial.py b/tutorials/tutorial21/tutorial.py index 8a539b1..3da40ff 100644 --- a/tutorials/tutorial21/tutorial.py +++ b/tutorials/tutorial21/tutorial.py @@ -3,7 +3,7 @@ # # Tutorial: Introductory Tutorial: Supervised Learning with PINA # -# [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial20/tutorial.ipynb) +# [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial21/tutorial.ipynb) # # # > ##### ⚠️ ***Before starting:*** diff --git a/tutorials/tutorial3/tutorial.py b/tutorials/tutorial3/tutorial.py index 1022d95..f35daac 100644 --- a/tutorials/tutorial3/tutorial.py +++ b/tutorials/tutorial3/tutorial.py @@ -2,11 +2,11 @@ # coding: utf-8 # # Tutorial: Applying Hard Constraints in PINNs to solve the Wave Problem -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial3/tutorial.ipynb) -# +# # In this tutorial, we will present how to solve the wave equation using **hard constraint Physics-Informed Neural Networks (PINNs)**. To achieve this, we will build a custom `torch` model and pass it to the **PINN solver**. -# +# # First of all, some useful imports. # In[ ]: @@ -37,10 +37,10 @@ from pina.callback import MetricTracker warnings.filterwarnings("ignore") -# ## The problem definition -# +# ## The problem definition +# # The problem is described by the following system of partial differential equations (PDEs): -# +# # \begin{equation} # \begin{cases} # \Delta u(x,y,t) = \frac{\partial^2}{\partial t^2} u(x,y,t) \quad \text{in } D, \\\\ @@ -48,9 +48,9 @@ warnings.filterwarnings("ignore") # u(x, y, t) = 0 \quad \text{on } \Gamma_1 \cup \Gamma_2 \cup \Gamma_3 \cup \Gamma_4, # \end{cases} # \end{equation} -# +# # Where: -# +# # - $D$ is a square domain $[0, 1]^2$. # - $\Gamma_i$, where $i = 1, \dots, 4$, are the boundaries of the square where Dirichlet conditions are applied. # - The velocity in the standard wave equation is fixed to $1$. @@ -111,13 +111,13 @@ problem = Wave() # ## Hard Constraint Model -# +# # Once the problem is defined, a **torch** model is needed to solve the PINN. While **PINA** provides several pre-implemented models, users have the option to build their own custom model using **torch**. The hard constraint we impose is on the boundary of the spatial domain. Specifically, the solution is written as: -# +# # $$ u_{\rm{pinn}} = xy(1-x)(1-y)\cdot NN(x, y, t), $$ -# +# # where $NN$ represents the neural network output. This neural network takes the spatial coordinates $x$, $y$, and time $t$ as input and provides the unknown field $u$. By construction, the solution is zero at the boundaries. -# +# # The residuals of the equations are evaluated at several sampling points (which the user can manipulate using the `discretise_domain` method). The loss function minimized by the neural network is the sum of the residuals. # In[3]: @@ -243,13 +243,13 @@ plot_solution(solver=pinn, time=1) # The results are not ideal, and we can clearly see that as time progresses, the solution deteriorates. Can we do better? -# +# # One valid approach is to impose the initial condition as a hard constraint as well. Specifically, we modify the solution to: -# +# # $$ # u_{\rm{pinn}} = xy(1-x)(1-y) \cdot NN(x, y, t) \cdot t + \cos(\sqrt{2}\pi t)\sin(\pi x)\sin(\pi y), # $$ -# +# # Now, let us start by building the neural network. # In[8]: @@ -329,19 +329,19 @@ plot_solution(solver=pinn, time=1) # We can now see that the results are much better! This improvement is due to the fact that, previously, the network was not correctly learning the initial condition, which led to a poor solution as time evolved. By imposing the initial condition as a hard constraint, the network is now able to correctly solve the problem. # ## What's Next? -# +# # Congratulations on completing the two-dimensional Wave tutorial of **PINA**! Now that you’ve got the basics down, there are several directions you can explore: -# +# # 1. **Train the Network for Longer**: Train the network for a longer duration or experiment with different layer sizes to assess the final accuracy. -# +# # 2. **Propose New Types of Hard Constraints in Time**: Experiment with new time-dependent hard constraints, for example: -# +# # $$ # u_{\rm{pinn}} = xy(1-x)(1-y)\cdot NN(x, y, t)(1-\exp(-t)) + \cos(\sqrt{2}\pi t)\sin(\pi x)\sin(\pi y) # $$ -# +# # 3. **Exploit Extrafeature Training**: Apply extrafeature training techniques to improve models from 1 and 2. -# +# # 4. **...and many more!**: The possibilities are endless! Keep experimenting and pushing the boundaries. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial4/tutorial.py b/tutorials/tutorial4/tutorial.py index 4b4fa39..e4fe405 100644 --- a/tutorials/tutorial4/tutorial.py +++ b/tutorials/tutorial4/tutorial.py @@ -5,7 +5,7 @@ # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial4/tutorial.ipynb) # In this tutorial, we will show how to use the Continuous Convolutional Filter, and how to build common Deep Learning architectures with it. The implementation of the filter follows the original work [*A Continuous Convolutional Trainable Filter for Modelling Unstructured Data*](https://arxiv.org/abs/2210.13416). -# +# # First of all we import the modules needed for the tutorial: # In[ ]: @@ -37,72 +37,72 @@ warnings.filterwarnings("ignore") # ## Tutorial Structure -# +# # The tutorial is structured as follows: -# -# - [🔹 Continuous Filter Background](#continuous-filter-background): +# +# - [🔹 Continuous Filter Background](#continuous-filter-background): # Understand how the convolutional filter works and how to use it. -# -# - [🔹 Building a MNIST Classifier](#building-a-mnist-classifier): +# +# - [🔹 Building a MNIST Classifier](#building-a-mnist-classifier): # Learn how to build a simple classifier using the MNIST dataset, and how to combine a continuous convolutional layer with a feedforward neural network. -# -# - [🔹 Building a Continuous Convolutional Autoencoder](#building-a-continuous-convolutional-autoencoder): +# +# - [🔹 Building a Continuous Convolutional Autoencoder](#building-a-continuous-convolutional-autoencoder): # Explore how to use the continuous filter to work with unstructured data for autoencoding and up-sampling. -# +# # ## Continuous Filter Background -# +# # As reported by the authors in the original paper, in contrast to discrete convolution, **continuous convolution** is mathematically defined as: -# +# # $$ # \mathcal{I}_{\rm{out}}(\mathbf{x}) = \int_{\mathcal{X}} \mathcal{I}(\mathbf{x} + \mathbf{\tau}) \cdot \mathcal{K}(\mathbf{\tau}) d\mathbf{\tau}, # $$ -# +# # where: # - $\mathcal{K} : \mathcal{X} \rightarrow \mathbb{R}$ is the **continuous filter** function, # - $\mathcal{I} : \Omega \subset \mathbb{R}^N \rightarrow \mathbb{R}$ is the input function. -# +# # The **continuous filter function** is approximated using a **FeedForward Neural Network**, which is **trainable** during the training phase. The way in which the integral is approximated can vary. In the **PINA** framework, we approximate it using a simple sum, as suggested by the authors. Thus, given the points $\{\mathbf{x}_i\}_{i=1}^{n}$ in $\mathbb{R}^N$ mapped onto the filter domain $\mathcal{X}$, we approximate the equation as: -# +# # $$ # \mathcal{I}_{\rm{out}}(\mathbf{\tilde{x}}_i) = \sum_{{\mathbf{x}_i}\in\mathcal{X}} \mathcal{I}(\mathbf{x}_i + \mathbf{\tau}) \cdot \mathcal{K}(\mathbf{x}_i), # $$ -# +# # where $\mathbf{\tau} \in \mathcal{S}$, with $\mathcal{S}$ being the set of available strides, represents the current stride position of the filter. The $\mathbf{\tilde{x}}_i$ points are obtained by taking the **centroid** of the filter position mapped onto the domain $\Omega$. -# +# # ### Working with the Continuous Filter -# +# # From the above definition, what is needed is: # 1. A **domain** and a **function** defined on that domain (the input), # 2. A **stride**, corresponding to the positions where the filter needs to be applied (this is the `stride` variable in `ContinuousConv`), # 3. The **filter's rectangular domain**, which corresponds to the `filter_dim` variable in `ContinuousConv`. -# +# # ### Input Function -# +# # The input function for the continuous filter is defined as a tensor of shape: -# +# # $$[B \times N_{\text{in}} \times N \times D]$$ -# +# # where: # - $B$ is the **batch size**, # - $N_{\text{in}}$ is the number of input fields, # - $N$ is the number of points in the mesh, -# - $D$ is the dimension of the problem. -# +# - $D$ is the dimension of the problem. +# # In particular: # - $D$ represents the **number of spatial variables** + 1. The last column must contain the field value. For example, for 2D problems, $D=3$ and the tensor will look like `[first coordinate, second coordinate, field value]`. # - $N_{\text{in}}$ represents the number of vectorial functions presented. For example, a vectorial function $f = [f_1, f_2]$ will have $N_{\text{in}}=2$. -# +# # #### Example: Input Function for a Vectorial Field -# +# # Let’s see an example to clarify the idea. Suppose we wish to create the function: -# +# # $$ # f(x, y) = [\sin(\pi x) \sin(\pi y), -\sin(\pi x) \sin(\pi y)] \quad (x,y)\in[0,1]\times[0,1] # $$ -# +# # We can do this with a **batch size** equal to 1. This function consists of two components (vectorial field), so $N_{\text{in}}=2$. For each $(x,y)$ pair in the domain $[0,1] \times [0,1]$, we will compute the corresponding field values: -# +# # 1. $\sin(\pi x) \sin(\pi y)$ # 2. $-\sin(\pi x) \sin(\pi y)$ @@ -139,9 +139,9 @@ print(f"Filter input data has shape: {data.shape}") # ### Stride -# +# # The **stride** is passed as a dictionary `stride` that dictates where the filter should move. Here's an example for the domain $[0,1] \times [0,5]$: -# +# # ```python # # stride definition # stride = {"domain": [1, 5], @@ -155,9 +155,9 @@ print(f"Filter input data has shape: {data.shape}") # 2. `start`: The starting position of the filter's centroid. In this example, the filter starts at the position $(0, 0)$. # 3. `jump`: The steps or jumps of the filter’s centroid to the next position. In this example, the filter moves by $(0.1, 0.3)$ along the x and y axes respectively. # 4. `direction`: The directions of the jumps for each coordinate. A value of 1 indicates the filter moves right, 0 means no movement, and -1 indicates the filter moves left with respect to its current position. -# +# # ### Filter definition -# +# # Now that we have defined the stride, we can move on to construct the continuous filter. # Let’s assume we want the output to contain only one field, and we will set the filter dimension to be $[0.1, 0.1]$. @@ -185,7 +185,7 @@ cConv = ContinuousConvBlock( # That's it! In just one line of code, we have successfully created the continuous convolutional filter. By default, the `pina.model.FeedForward` neural network is initialized, which can be further customized according to your needs. -# +# # Additionally, if the mesh does not change during training, we can set the `optimize` flag to `True` to leverage optimizations for efficiently finding the points to convolve. This feature helps in improving the performance by reducing redundant calculations when the mesh remains constant. # In[4]: @@ -244,10 +244,10 @@ cConv = ContinuousConvBlock( ) -# Notice that we pass the **class** of the model and not an already built object! This is important because the `ContinuousConv` filter will automatically instantiate the model class when needed during training. -# +# Notice that we pass the **class** of the model and not an already built object! This is important because the `ContinuousConv` filter will automatically instantiate the model class when needed during training. +# # ## Building a MNIST Classifier -# +# # Let's see how we can build a MNIST classifier using a continuous convolutional filter. We will use the MNIST dataset from PyTorch. In order to keep small training times we use only 6000 samples for training and 1000 samples for testing. # In[7]: @@ -276,7 +276,7 @@ train_data = torchvision.datasets.MNIST( # Now, let's proceed to build a simple classifier for the MNIST dataset. The MNIST dataset consists of vectors with the shape `[batch, 1, 28, 28]`, but we can treat them as field functions where each pixel at coordinates $i,j$ corresponds to a point in a $[0, 27] \times [0, 27]$ domain. The pixel values represent the field values. -# +# # To use the continuous convolutional filter, we need to transform the regular tensor into a format compatible with the filter. Here's a function that will help with this transformation: # In[8]: @@ -401,9 +401,9 @@ print(f"Accuracy of the network on the test images: {(correct / total):.3%}") # As we can see we have very good performance for having trained only for 1 epoch! Nevertheless, we are still using structured data... Let's see how we can build an autoencoder for unstructured data now. -# +# # ## Building a Continuous Convolutional Autoencoder -# +# # As a toy problem, we will now build an autoencoder for the function \( f(x, y) = \sin(\pi x) \sin(\pi y) \) on the unit circle domain centered at \( (0.5, 0.5) \). We will also explore the ability to up-sample the results (once trained) without needing to retrain the model. To begin, we'll generate the input data for the function. First, we will use a mesh of 100 points and visualize the input function. Here’s how to proceed: # In[12]: @@ -451,7 +451,7 @@ plt.show() # Now, let's create a simple autoencoder using the continuous convolutional filter. Since the data is inherently unstructured, a standard convolutional filter may not be effective without some form of projection or interpolation. We'll begin by building an `Encoder` and `Decoder` class, and then combine them into a unified `Autoencoder` class. -# +# # In[13]: @@ -608,9 +608,9 @@ print(f"l2 error: {l2_error(input_data[0, 0, :, -1], output[0, 0, :, -1]):.2%}") # The $l_2$ error is approximately $4\%$, which is quite low considering that we only use **one** convolutional layer and a simple feedforward network to reduce the dimension. Now, let's explore some of the unique features of the filter. -# +# # ### Upsampling with the Filter -# +# # Suppose we have a hidden representation and we want to upsample it on a different grid with more points. Let's see how we can achieve that: # In[18]: @@ -656,15 +656,15 @@ print( # ## What's Next? -# +# # Congratulations on completing the tutorial on using the Continuous Convolutional Filter in **PINA**! Now that you have the basics, there are several exciting directions you can explore: -# +# # 1. **Train using Physics-Informed strategies**: Leverage physics-based knowledge to improve model performance for solving real-world problems. -# +# # 2. **Use the filter to build an unstructured convolutional autoencoder**: Explore reduced-order modeling by implementing unstructured convolutional autoencoders. -# +# # 3. **Experiment with upsampling at different resolutions**: Try encoding or upsampling on different grids to see how the model generalizes across multiple resolutions. -# +# # 4. **...and many more!**: There are endless possibilities, from improving model architecture to testing with more complex datasets. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial5/tutorial.py b/tutorials/tutorial5/tutorial.py index f1339ad..1231ccb 100644 --- a/tutorials/tutorial5/tutorial.py +++ b/tutorials/tutorial5/tutorial.py @@ -2,13 +2,13 @@ # coding: utf-8 # # Tutorial: Modeling 2D Darcy Flow with the Fourier Neural Operator -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial5/tutorial.ipynb) -# +# # In this tutorial, we are going to solve the **Darcy flow problem** in two dimensions, as presented in the paper [*Fourier Neural Operator for Parametric Partial Differential Equations*](https://openreview.net/pdf?id=c8P9NQVtmnO). -# +# # We begin by importing the necessary modules for the tutorial: -# +# # In[ ]: @@ -22,9 +22,11 @@ except: IN_COLAB = False if IN_COLAB: get_ipython().system('pip install "pina-mathlab[tutorial]"') - get_ipython().system('pip install scipy') + get_ipython().system("pip install scipy") # get the data - get_ipython().system('wget https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial5/Data_Darcy.mat') + get_ipython().system( + "wget https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial5/Data_Darcy.mat" + ) import torch import matplotlib.pyplot as plt @@ -40,15 +42,15 @@ warnings.filterwarnings("ignore") # ## Data Generation -# +# # We will focus on solving a specific PDE: the **Darcy Flow** equation. This is a second-order elliptic PDE given by: -# +# # $$ # -\nabla\cdot(k(x, y)\nabla u(x, y)) = f(x, y), \quad (x, y) \in D. # $$ -# +# # Here, $u$ represents the flow pressure, $k$ is the permeability field, and $f$ is the forcing function. The Darcy flow equation can be used to model various systems, including flow through porous media, elasticity in materials, and heat conduction. -# +# # In this tutorial, the domain $D$ is defined as a 2D unit square with Dirichlet boundary conditions. The dataset used is taken from the authors' original implementation in the referenced paper. # In[2]: @@ -92,7 +94,7 @@ problem = SupervisedProblem( # ## Solving the Problem with a Feedforward Neural Network -# +# # We begin by solving the Darcy flow problem using a standard Feedforward Neural Network (FNN). Since we are approaching this task with supervised learning, we will use the `SupervisedSolver` provided by **PINA** to train the model. # In[7]: @@ -146,7 +148,7 @@ print(f"Final error testing {err:.2f}%") # ## Solving the Problem with a Fourier Neural Operator -# +# # We will now solve the Darcy flow problem using a Fourier Neural Operator (FNO). Since we are learning a mapping between functions—i.e., an operator—this approach is more suitable and often yields better performance, as we will see. # In[10]: @@ -183,7 +185,7 @@ trainer.train() # We can clearly observe that the final loss is significantly lower when using the FNO. Let's now evaluate its performance on the test set. -# +# # Note that the number of trainable parameters in the FNO is considerably higher compared to a `FeedForward` network. Therefore, we recommend using a GPU or TPU to accelerate training, especially when working with large datasets. # In[11]: @@ -208,13 +210,13 @@ print(f"Final error testing {err:.2f}%") # As we can see, the loss is significantly lower with the Fourier Neural Operator! # ## What's Next? -# +# # Congratulations on completing the tutorial on solving the Darcy flow problem using **PINA**! There are many potential next steps you can explore: -# +# # 1. **Train the network longer or with different hyperparameters**: Experiment with different configurations of the neural network. You can try varying the number of layers, activation functions, or learning rates to improve accuracy. -# +# # 2. **Solve more complex problems**: The Darcy flow problem is just the beginning! Try solving other complex problems from the field of parametric PDEs. The original paper and **PINA** documentation offer many more examples to explore. -# +# # 3. **...and many more!**: There are countless directions to further explore. For instance, you could try to add physics informed learning! -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial6/tutorial.py b/tutorials/tutorial6/tutorial.py index 80dc579..4f9d6fe 100644 --- a/tutorials/tutorial6/tutorial.py +++ b/tutorials/tutorial6/tutorial.py @@ -2,15 +2,15 @@ # coding: utf-8 # # Tutorial: Building geometries with PINA `DomainInterface` class -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial6/tutorial.ipynb) -# +# # In this tutorial we will show how to use geometries in PINA. Specifically, the tutorial will include how to create geometries and how to visualize them. The topics covered are: -# +# # * Creating CartesianDomains and EllipsoidDomains # * Getting the Union and Difference of Geometries # * Sampling points in the domain (and visualize them) -# +# # We import the relevant modules first. # In[ ]: @@ -60,8 +60,8 @@ ellipsoid_border = EllipsoidDomain( ) -# The `{'x': [0, 2], 'y': [0, 2]}` are the bounds of the `CartesianDomain` being created. -# +# The `{'x': [0, 2], 'y': [0, 2]}` are the bounds of the `CartesianDomain` being created. +# # To visualize these shapes, we need to sample points on them. We will use the `sample` method of the `CartesianDomain` and `EllipsoidDomain` classes. This method takes a `n` argument which is the number of points to sample. It also takes different modes to sample, such as `'random'`. # In[3]: @@ -101,7 +101,7 @@ for ax, pts, title in zip(axs, pts_list, title_list): # We have now created, sampled, and visualized our first geometries! We can see that the `EllipsoidDomain` with the border has a border around it. We can also see that the `EllipsoidDomain` without the border is just the ellipse. We can also see that the `CartesianDomain` is just a square. # ### Simplex Domain -# +# # Among the built-in shapes, we quickly show here the usage of `SimplexDomain`, which can be used for polygonal domains! # In[6]: @@ -135,7 +135,7 @@ for domain, ax in zip([spatial_domain, spatial_domain2], axs): # ## Boolean Operations # To create complex shapes we can use the boolean operations, for example to merge two default geometries. We need to simply use the `Union` class: it takes a list of geometries and returns the union of them. -# +# # Let's create three unions. Firstly, it will be a union of `cartesian` and `ellipsoid_no_border`. Next, it will be a union of `ellipse_no_border` and `ellipse_border`. Lastly, it will be a union of all three geometries. # In[7]: @@ -224,7 +224,7 @@ class Heart(DomainInterface): pass -# Now we have the skeleton for our `Heart` class. Also the `sample` method is where most of the work is done so let's fill it out. +# Now we have the skeleton for our `Heart` class. Also the `sample` method is where most of the work is done so let's fill it out. # In[13]: @@ -278,13 +278,13 @@ plot_scatter(ax, pts_heart, "Heart Domain") # ## What's Next? -# +# # We have walked through a simple tutorial on how to build custom geometries and use domain operations to compose base geometries. Now you can experiment with different geometries and create your own! -# +# # 1. **Experiment with Complex Geometries**: Combine multiple basic shapes to create more intricate structures using domain operations. -# +# # 2. **Optimize Geometry for Specific Tasks**: Customize your geometry models for specialized applications such as fluid dynamics, heat transfer, or structural analysis. -# +# # 3. **...and many more!**: Explore new geometries and build them with `DomainInterface`! -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial7/tutorial.py b/tutorials/tutorial7/tutorial.py index e70cea0..c7030a4 100644 --- a/tutorials/tutorial7/tutorial.py +++ b/tutorials/tutorial7/tutorial.py @@ -2,15 +2,15 @@ # coding: utf-8 # # Tutorial: Inverse Problem Solving with Physics-Informed Neural Network -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial7/tutorial.ipynb) -# +# # ## Introduction to the Inverse Problem -# +# # This tutorial demonstrates how to solve an inverse Poisson problem using Physics-Informed Neural Networks (PINNs). -# +# # The problem is defined as a Poisson equation with homogeneous boundary conditions: -# +# # \begin{equation} # \begin{cases} # \Delta u = e^{-2(x - \mu_1)^2 - 2(y - \mu_2)^2} \quad \text{in } \Omega, \\ @@ -18,18 +18,18 @@ # u(\mu_1, \mu_2) = \text{data} # \end{cases} # \end{equation} -# +# # Here, $\Omega$ is the square domain $[-2, 2] \times [-2, 2]$, and $\partial \Omega = \Gamma_1 \cup \Gamma_2 \cup \Gamma_3 \cup \Gamma_4$ represents the union of its boundaries. -# +# # This type of setup defines an *inverse problem*, which has two primary objectives: -# +# # - **Find the solution** $u$ that satisfies the Poisson equation, # - **Identify the unknown parameters** $(\mu_1, \mu_2)$ that best fit the given data (as described by the third equation in the system). -# +# # To tackle both objectives, we will define an `InverseProblem` using **PINA**. -# +# # Let's begin with the necessary imports: -# +# # In[ ]: @@ -45,8 +45,12 @@ if IN_COLAB: get_ipython().system('pip install "pina-mathlab[tutorial]"') # get the data get_ipython().system('mkdir "data"') - get_ipython().system('wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial7/data/pinn_solution_0.5_0.5" -O "data/pinn_solution_0.5_0.5"') - get_ipython().system('wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial7/data/pts_0.5_0.5" -O "data/pts_0.5_0.5"') + get_ipython().system( + 'wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial7/data/pinn_solution_0.5_0.5" -O "data/pinn_solution_0.5_0.5"' + ) + get_ipython().system( + 'wget "https://github.com/mathLab/PINA/raw/refs/heads/master/tutorials/tutorial7/data/pts_0.5_0.5" -O "data/pts_0.5_0.5"' + ) import matplotlib.pyplot as plt import torch @@ -68,14 +72,14 @@ warnings.filterwarnings("ignore") seed_everything(883) -# Next, we import the pre-saved data corresponding to the true parameter values $(\mu_1, \mu_2) = (0.5, 0.5)$. +# Next, we import the pre-saved data corresponding to the true parameter values $(\mu_1, \mu_2) = (0.5, 0.5)$. # These values represent the *optimal parameters* that we aim to recover through neural network training. -# +# # In particular, we load: -# +# # - `input` points — the spatial coordinates where observations are available, # - `target` points — the corresponding $u$ values (i.e., the solution evaluated at the `input` points). -# +# # This data will be used to guide the inverse problem and supervise the network’s prediction of the unknown parameters. # In[11]: @@ -88,10 +92,10 @@ data_input = torch.load("data/pts_0.5_0.5", weights_only=False) # Next, let's visualize the data: -# +# # - We'll plot the data points, i.e., the spatial coordinates where measurements are available. # - We'll also display the reference solution corresponding to $(\mu_1, \mu_2) = (0.5, 0.5)$. -# +# # This serves as the ground truth or expected output that our neural network should learn to approximate through training. # In[12]: @@ -107,10 +111,10 @@ plt.show() # ## Inverse Problem Definition in PINA -# -# Next, we initialize the Poisson problem, which inherits from the `SpatialProblem` and `InverseProblem` classes. +# +# Next, we initialize the Poisson problem, which inherits from the `SpatialProblem` and `InverseProblem` classes. # In this step, we need to define all the variables and specify the domain in which our unknown parameters $(\mu_1, \mu_2)$ reside. -# +# # Note that the Laplace equation also takes these unknown parameters as inputs. These parameters will be treated as variables that the neural network will optimize during the training process, enabling it to learn the optimal values for $(\mu_1, \mu_2)$. # In[13]: @@ -189,11 +193,11 @@ problem.discretise_domain( ) -# Here, we define a simple callback for the trainer. This callback is used to save the parameters predicted by the neural network during training. +# Here, we define a simple callback for the trainer. This callback is used to save the parameters predicted by the neural network during training. # The parameters are saved every 100 epochs as `torch` tensors in a specified directory (in our case, `tutorial_logs`). -# +# # The goal of this setup is to read the saved parameters after training and visualize their trend across the epochs. This allows us to monitor how the predicted parameters evolve throughout the training process. -# +# # In[16]: @@ -266,13 +270,13 @@ plt.show() # ## What's Next? -# +# # We have covered the basic usage of PINNs for inverse problem modeling. Here are some possible directions for further exploration: -# +# # 1. **Experiment with different Physics-Informed strategies**: Explore variations in PINN training techniques to improve performance or tackle different types of problems. -# +# # 2. **Apply to more complex problems**: Scale the approach to higher-dimensional or time-dependent inverse problems. -# +# # 3. **...and many more!**: The possibilities are endless, from integrating additional physical constraints to testing on real-world datasets. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/). diff --git a/tutorials/tutorial8/tutorial.py b/tutorials/tutorial8/tutorial.py index 77ea093..b873226 100644 --- a/tutorials/tutorial8/tutorial.py +++ b/tutorials/tutorial8/tutorial.py @@ -3,7 +3,7 @@ # # Tutorial: Reduced Order Modeling with POD-RBF and POD-NN Approaches for Fluid Dynamics # -# [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial9/tutorial.ipynb) +# [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial8/tutorial.ipynb) # The goal of this tutorial is to demonstrate how to use the **PINA** library to apply a reduced-order modeling technique, as outlined in [1]. These methods share several similarities with machine learning approaches, as they focus on predicting the solution to differential equations, often parametric PDEs, in real-time. # diff --git a/tutorials/tutorial9/tutorial.py b/tutorials/tutorial9/tutorial.py index 66c3272..5901166 100644 --- a/tutorials/tutorial9/tutorial.py +++ b/tutorials/tutorial9/tutorial.py @@ -2,14 +2,14 @@ # coding: utf-8 # # Tutorial: Applying Periodic Boundary Conditions in PINNs to solve the Helmotz Problem -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mathLab/PINA/blob/master/tutorials/tutorial9/tutorial.ipynb) -# -# This tutorial demonstrates how to solve a one-dimensional Helmholtz equation with periodic boundary conditions (PBC) using Physics-Informed Neural Networks (PINNs). +# +# This tutorial demonstrates how to solve a one-dimensional Helmholtz equation with periodic boundary conditions (PBC) using Physics-Informed Neural Networks (PINNs). # We will use standard PINN training, augmented with a periodic input expansion as introduced in [*An Expert’s Guide to Training Physics-Informed Neural Networks*](https://arxiv.org/abs/2308.08468). -# +# # Let's start with some useful imports: -# +# # In[1]: @@ -42,33 +42,33 @@ warnings.filterwarnings("ignore") # ## Problem Definition -# +# # The one-dimensional Helmholtz problem is mathematically expressed as: -# +# # $$ # \begin{cases} # \frac{d^2}{dx^2}u(x) - \lambda u(x) - f(x) &= 0 \quad \text{for } x \in (0, 2) \\ # u^{(m)}(x = 0) - u^{(m)}(x = 2) &= 0 \quad \text{for } m \in \{0, 1, \dots\} # \end{cases} # $$ -# -# In this case, we seek a solution that is $C^{\infty}$ (infinitely differentiable) and periodic with period 2, over the infinite domain $x \in (-\infty, \infty)$. -# +# +# In this case, we seek a solution that is $C^{\infty}$ (infinitely differentiable) and periodic with period 2, over the infinite domain $x \in (-\infty, \infty)$. +# # A classical PINN approach would require enforcing periodic boundary conditions (PBC) for all derivatives—an infinite set of constraints—which is clearly infeasible. -# +# # To address this, we adopt a strategy known as *coordinate augmentation*. In this approach, we apply a coordinate transformation $v(x)$ such that the transformed inputs naturally satisfy the periodicity condition: -# +# # $$ # u^{(m)}(x = 0) - u^{(m)}(x = 2) = 0 \quad \text{for } m \in \{0, 1, \dots\} # $$ -# +# # For demonstration purposes, we choose the specific parameters: -# +# # - $\lambda = -10\pi^2$ # - $f(x) = -6\pi^2 \sin(3\pi x) \cos(\pi x)$ -# +# # These yield an analytical solution: -# +# # $$ # u(x) = \sin(\pi x) \cos(3\pi x) # $$ @@ -111,39 +111,39 @@ problem.discretise_domain(200, "grid", domains=["phys_cond"]) # As usual, the Helmholtz problem is implemented in **PINA** as a class. The governing equations are defined as `conditions`, which must be satisfied within their respective domains. The `solution` represents the exact analytical solution, which will be used to evaluate the accuracy of the predicted solution. -# -# For selecting collocation points, we use Latin Hypercube Sampling (LHS), a common strategy for efficient space-filling in high-dimensional domains -# +# +# For selecting collocation points, we use Latin Hypercube Sampling (LHS), a common strategy for efficient space-filling in high-dimensional domains +# # ## Solving the Problem with a Periodic Network -# -# Any $\mathcal{C}^{\infty}$ periodic function $u : \mathbb{R} \rightarrow \mathbb{R}$ with period $L \in \mathbb{N}$ +# +# Any $\mathcal{C}^{\infty}$ periodic function $u : \mathbb{R} \rightarrow \mathbb{R}$ with period $L \in \mathbb{N}$ # can be constructed by composing an arbitrary smooth function $f : \mathbb{R}^n \rightarrow \mathbb{R}$ with a smooth, periodic mapping$v : \mathbb{R} \rightarrow \mathbb{R}^n$ of the same period $L$. That is, -# +# # $$ # u(x) = f(v(x)). # $$ -# -# This formulation is general and can be extended to arbitrary dimensions. +# +# This formulation is general and can be extended to arbitrary dimensions. # For more details, see [*A Method for Representing Periodic Functions and Enforcing Exactly Periodic Boundary Conditions with Deep Neural Networks*](https://arxiv.org/pdf/2007.07442). -# +# # In our specific case, we define the periodic embedding as: -# +# # $$ # v(x) = \left[1, \cos\left(\frac{2\pi}{L} x\right), \sin\left(\frac{2\pi}{L} x\right)\right], # $$ -# +# # which constitutes the coordinate augmentation. The function $f(\cdot)$ is approximated by a neural network $NN_{\theta}(\cdot)$, resulting in the approximate PINN solution: -# +# # $$ # u(x) \approx u_{\theta}(x) = NN_{\theta}(v(x)). # $$ -# -# In **PINA**, this is implemented using the `PeriodicBoundaryEmbedding` layer for $v(x)$, -# paired with any `pina.model` to define the neural network $NN_{\theta}$. -# +# +# In **PINA**, this is implemented using the `PeriodicBoundaryEmbedding` layer for $v(x)$, +# paired with any `pina.model` to define the neural network $NN_{\theta}$. +# # Let’s see how this is put into practice! -# -# +# +# # In[18]: @@ -160,11 +160,11 @@ model = torch.nn.Sequential( # As simple as that! -# -# In higher dimensions, you can specify different periods for each coordinate using a dictionary. -# For example, `periods = {'x': 2, 'y': 3, ...}` indicates a periodicity of 2 in the $x$ direction, +# +# In higher dimensions, you can specify different periods for each coordinate using a dictionary. +# For example, `periods = {'x': 2, 'y': 3, ...}` indicates a periodicity of 2 in the $x$ direction, # 3 in the $y$ direction, and so on. -# +# # We will now solve the problem using the usual `PINN` and `Trainer` classes. After training, we'll examine the losses using the `MetricTracker` callback from `pina.callback`. # In[ ]: @@ -240,15 +240,15 @@ with torch.no_grad(): # It's clear that the network successfully captures the periodicity of the solution, with the error also exhibiting a periodic pattern. Naturally, training for a longer duration or using a more expressive neural network could further improve the results. # ## What's next? -# +# # Congratulations on completing the one-dimensional Helmholtz tutorial with **PINA**! Here are a few directions you can explore next: -# +# # 1. **Train longer or with different architectures**: Experiment with extended training or modify the network's depth and width to evaluate improvements in accuracy. -# +# # 2. **Apply `PeriodicBoundaryEmbedding` to time-dependent problems**: Explore more complex scenarios such as spatiotemporal PDEs (see the official documentation for examples). -# +# # 3. **Try extra feature training**: Integrate additional physical or domain-specific features to guide the learning process more effectively. -# +# # 4. **...and many more!**: Extend to higher dimensions, test on other PDEs, or even develop custom embeddings tailored to your problem. -# +# # For more resources and tutorials, check out the [PINA Documentation](https://mathlab.github.io/PINA/).