diff --git a/docs/source/_rst/_code.rst b/docs/source/_rst/_code.rst index 74c7852..77072a5 100644 --- a/docs/source/_rst/_code.rst +++ b/docs/source/_rst/_code.rst @@ -74,7 +74,27 @@ Layers Continuous convolution Proper Orthogonal Decomposition Periodic Boundary Condition embeddings - Adpative Activation Function + +Adaptive Activation Functions +------------------------------- + +.. toctree:: + :titlesonly: + + Adaptive Function Interface + Adaptive ReLU + Adaptive Sigmoid + Adaptive Tanh + Adaptive SiLU + Adaptive Mish + Adaptive ELU + Adaptive CELU + Adaptive GELU + Adaptive Softmin + Adaptive Softmax + Adaptive SIREN + Adaptive Exp + Equations and Operators ------------------------- diff --git a/docs/source/_rst/adaptive_functions/AdaptiveCELU.rst b/docs/source/_rst/adaptive_functions/AdaptiveCELU.rst new file mode 100644 index 0000000..9736ee6 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveCELU.rst @@ -0,0 +1,9 @@ +AdaptiveCELU +============ + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveCELU + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveELU.rst b/docs/source/_rst/adaptive_functions/AdaptiveELU.rst new file mode 100644 index 0000000..ad04717 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveELU.rst @@ -0,0 +1,9 @@ +AdaptiveELU +=========== + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveELU + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveExp.rst b/docs/source/_rst/adaptive_functions/AdaptiveExp.rst new file mode 100644 index 0000000..7d07cd5 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveExp.rst @@ -0,0 +1,9 @@ +AdaptiveExp +=========== + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveExp + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveFunctionInterface.rst b/docs/source/_rst/adaptive_functions/AdaptiveFunctionInterface.rst new file mode 100644 index 0000000..7cdf754 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveFunctionInterface.rst @@ -0,0 +1,8 @@ +AdaptiveActivationFunctionInterface +======================================= + +.. currentmodule:: pina.adaptive_functions.adaptive_func_interface + +.. automodule:: pina.adaptive_functions.adaptive_func_interface + :members: + :show-inheritance: diff --git a/docs/source/_rst/adaptive_functions/AdaptiveGELU.rst b/docs/source/_rst/adaptive_functions/AdaptiveGELU.rst new file mode 100644 index 0000000..86e5875 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveGELU.rst @@ -0,0 +1,9 @@ +AdaptiveGELU +============ + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveGELU + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveMish.rst b/docs/source/_rst/adaptive_functions/AdaptiveMish.rst new file mode 100644 index 0000000..4e1e3b4 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveMish.rst @@ -0,0 +1,9 @@ +AdaptiveMish +============ + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveMish + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveReLU.rst b/docs/source/_rst/adaptive_functions/AdaptiveReLU.rst new file mode 100644 index 0000000..ea08c29 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveReLU.rst @@ -0,0 +1,9 @@ +AdaptiveReLU +============ + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveReLU + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveSIREN.rst b/docs/source/_rst/adaptive_functions/AdaptiveSIREN.rst new file mode 100644 index 0000000..96133bd --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveSIREN.rst @@ -0,0 +1,9 @@ +AdaptiveSIREN +============= + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveSIREN + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveSiLU.rst b/docs/source/_rst/adaptive_functions/AdaptiveSiLU.rst new file mode 100644 index 0000000..2f359fd --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveSiLU.rst @@ -0,0 +1,9 @@ +AdaptiveSiLU +============ + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveSiLU + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveSigmoid.rst b/docs/source/_rst/adaptive_functions/AdaptiveSigmoid.rst new file mode 100644 index 0000000..6f495a8 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveSigmoid.rst @@ -0,0 +1,9 @@ +AdaptiveSigmoid +=============== + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveSigmoid + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveSoftmax.rst b/docs/source/_rst/adaptive_functions/AdaptiveSoftmax.rst new file mode 100644 index 0000000..5cab9c6 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveSoftmax.rst @@ -0,0 +1,9 @@ +AdaptiveSoftmax +=============== + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveSoftmax + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveSoftmin.rst b/docs/source/_rst/adaptive_functions/AdaptiveSoftmin.rst new file mode 100644 index 0000000..a0e6c94 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveSoftmin.rst @@ -0,0 +1,9 @@ +AdaptiveSoftmin +=============== + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveSoftmin + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/adaptive_functions/AdaptiveTanh.rst b/docs/source/_rst/adaptive_functions/AdaptiveTanh.rst new file mode 100644 index 0000000..3e48651 --- /dev/null +++ b/docs/source/_rst/adaptive_functions/AdaptiveTanh.rst @@ -0,0 +1,9 @@ +AdaptiveTanh +============ + +.. currentmodule:: pina.adaptive_functions.adaptive_func + +.. autoclass:: AdaptiveTanh + :members: + :show-inheritance: + :inherited-members: AdaptiveActivationFunctionInterface diff --git a/docs/source/_rst/layers/adaptive_func.rst b/docs/source/_rst/layers/adaptive_func.rst deleted file mode 100644 index 278f9f7..0000000 --- a/docs/source/_rst/layers/adaptive_func.rst +++ /dev/null @@ -1,7 +0,0 @@ -AdaptiveActivationFunction -============================= -.. currentmodule:: pina.model.layers.adaptive_func - -.. autoclass:: AdaptiveActivationFunction - :members: - :show-inheritance: \ No newline at end of file diff --git a/pina/adaptive_functions/__init__.py b/pina/adaptive_functions/__init__.py new file mode 100644 index 0000000..0ab6053 --- /dev/null +++ b/pina/adaptive_functions/__init__.py @@ -0,0 +1,21 @@ +__all__ = [ + 'AdaptiveActivationFunctionInterface', + 'AdaptiveReLU', + 'AdaptiveSigmoid', + 'AdaptiveTanh', + 'AdaptiveSiLU', + 'AdaptiveMish', + 'AdaptiveELU', + 'AdaptiveCELU', + 'AdaptiveGELU', + 'AdaptiveSoftmin', + 'AdaptiveSoftmax', + 'AdaptiveSIREN', + 'AdaptiveExp'] + +from .adaptive_func import (AdaptiveReLU, AdaptiveSigmoid, AdaptiveTanh, + AdaptiveSiLU, AdaptiveMish, AdaptiveELU, + AdaptiveCELU, AdaptiveGELU, AdaptiveSoftmin, + AdaptiveSoftmax, AdaptiveSIREN, AdaptiveExp) +from .adaptive_func_interface import AdaptiveActivationFunctionInterface + diff --git a/pina/adaptive_functions/adaptive_func.py b/pina/adaptive_functions/adaptive_func.py new file mode 100644 index 0000000..0ee22b2 --- /dev/null +++ b/pina/adaptive_functions/adaptive_func.py @@ -0,0 +1,488 @@ +""" Module for adaptive functions. """ + +import torch +from ..utils import check_consistency +from .adaptive_func_interface import AdaptiveActivationFunctionInterface + + +class AdaptiveReLU(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.ReLU` activation function. + + Given the function :math:`\text{ReLU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{ReLU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{ReLU}_{\text{adaptive}}({x}) = \alpha\,\text{ReLU}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + ReLU function is defined as: + + .. math:: + \text{ReLU}(x) = \max(0, x) + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.ReLU() + + +class AdaptiveSigmoid(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.Sigmoid` activation function. + + Given the function :math:`\text{Sigmoid}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{Sigmoid}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Sigmoid}_{\text{adaptive}}({x}) = \alpha\,\text{Sigmoid}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + Sigmoid function is defined as: + + .. math:: + \text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)} + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Sigmoid() + + +class AdaptiveTanh(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.Tanh` activation function. + + Given the function :math:`\text{Tanh}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{Tanh}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Tanh}_{\text{adaptive}}({x}) = \alpha\,\text{Tanh}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + Tanh function is defined as: + + .. math:: + \text{Tanh}(x) = \frac{\exp(x) - \exp(-x)} {\exp(x) + \exp(-x)} + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Tanh() + + +class AdaptiveSiLU(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.SiLU` activation function. + + Given the function :math:`\text{SiLU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{SiLU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{SiLU}_{\text{adaptive}}({x}) = \alpha\,\text{SiLU}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + SiLU function is defined as: + + .. math:: + \text{SiLU}(x) = x * \sigma(x), \text{where }\sigma(x) + \text{ is the logistic sigmoid.} + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.SiLU() + + +class AdaptiveMish(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.Mish` activation function. + + Given the function :math:`\text{Mish}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{Mish}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Mish}_{\text{adaptive}}({x}) = \alpha\,\text{Mish}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + Mish function is defined as: + + .. math:: + \text{Mish}(x) = x * \text{Tanh}(x) + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Mish() + + +class AdaptiveELU(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.ELU` activation function. + + Given the function :math:`\text{ELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{ELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{ELU}_{\text{adaptive}}({x}) = \alpha\,\text{ELU}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + ELU function is defined as: + + .. math:: + \text{ELU}(x) = \begin{cases} + x, & \text{ if }x > 0\\ + \exp(x) - 1, & \text{ if }x \leq 0 + \end{cases} + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.ELU() + + +class AdaptiveCELU(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.CELU` activation function. + + Given the function :math:`\text{CELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{CELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{CELU}_{\text{adaptive}}({x}) = \alpha\,\text{CELU}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + CELU function is defined as: + + .. math:: + \text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x) - 1)) + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.CELU() + +class AdaptiveGELU(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.GELU` activation function. + + Given the function :math:`\text{GELU}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{GELU}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{GELU}_{\text{adaptive}}({x}) = \alpha\,\text{GELU}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + GELU function is defined as: + + .. math:: + \text{GELU}(x) = 0.5 * x * (1 + \text{Tanh}(\sqrt{2 / \pi} * (x + 0.044715 * x^3))) + + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.GELU() + + +class AdaptiveSoftmin(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.Softmin` activation function. + + Given the function :math:`\text{Softmin}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{Softmin}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Softmin}_{\text{adaptive}}({x}) = \alpha\,\text{Softmin}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + Softmin function is defined as: + + .. math:: + \text{Softmin}(x_{i}) = \frac{\exp(-x_i)}{\sum_j \exp(-x_j)} + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Softmin() + + +class AdaptiveSoftmax(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :class:`~torch.nn.Softmax` activation function. + + Given the function :math:`\text{Softmax}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{Softmax}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{Softmax}_{\text{adaptive}}({x}) = \alpha\,\text{Softmax}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters, and the + Softmax function is defined as: + + .. math:: + \text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)} + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.nn.Softmax() + +class AdaptiveSIREN(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :obj:`~torch.sin` function. + + Given the function :math:`\text{sin}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{sin}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{sin}_{\text{adaptive}}({x}) = \alpha\,\text{sin}(\beta{x}+\gamma), + + where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters. + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): + super().__init__(alpha, beta, gamma, fixed) + self._func = torch.sin + +class AdaptiveExp(AdaptiveActivationFunctionInterface): + r""" + Adaptive trainable :obj:`~torch.exp` function. + + Given the function :math:`\text{exp}:\mathbb{R}^n\rightarrow\mathbb{R}^n`, + the adaptive function + :math:`\text{exp}_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^n` + is defined as: + + .. math:: + \text{exp}_{\text{adaptive}}({x}) = \alpha\,\text{exp}(\beta{x}), + + where :math:`\alpha,\,\beta` are trainable parameters. + + .. seealso:: + + **Original reference**: Godfrey, Luke B., and Michael S. Gashler. + *A continuum among logarithmic, linear, and exponential functions, + and its potential to improve generalization in neural networks.* + 2015 7th international joint conference on knowledge discovery, + knowledge engineering and knowledge management (IC3K). + Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. + `_. + + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. + """ + def __init__(self, alpha=None, beta=None, fixed=None): + + # only alpha, and beta parameters (gamma=0 fixed) + if fixed is None: + fixed = ['gamma'] + else: + check_consistency(fixed, str) + fixed = list(fixed) + ['gamma'] + + # calling super + super().__init__(alpha, beta, 0., fixed) + self._func = torch.exp \ No newline at end of file diff --git a/pina/model/layers/adaptive_func.py b/pina/adaptive_functions/adaptive_func_interface.py similarity index 68% rename from pina/model/layers/adaptive_func.py rename to pina/adaptive_functions/adaptive_func_interface.py index 3bfc4be..b0522d5 100644 --- a/pina/model/layers/adaptive_func.py +++ b/pina/adaptive_functions/adaptive_func_interface.py @@ -1,14 +1,18 @@ """ Module for adaptive functions. """ import torch + from pina.utils import check_consistency +from abc import ABCMeta -class AdaptiveActivationFunction(torch.nn.Module): +class AdaptiveActivationFunctionInterface(torch.nn.Module, metaclass=ABCMeta): r""" - The :class:`~pina.model.layers.adaptive_func.AdaptiveActivationFunction` + The + :class:`~pina.adaptive_functions.adaptive_func_interface.AdaptiveActivationFunctionInterface` class makes a :class:`torch.nn.Module` activation function into an adaptive - trainable activation function. + trainable activation function. If one wants to create an adpative activation + function, this class must be use as base class. Given a function :math:`f:\mathbb{R}^n\rightarrow\mathbb{R}^m`, the adaptive function :math:`f_{\text{adaptive}}:\mathbb{R}^n\rightarrow\mathbb{R}^m` @@ -19,28 +23,6 @@ class AdaptiveActivationFunction(torch.nn.Module): where :math:`\alpha,\,\beta,\,\gamma` are trainable parameters. - :Example: - >>> import torch - >>> from pina.model.layers import AdaptiveActivationFunction - >>> - >>> # simple adaptive function with all trainable parameters - >>> AdaptiveTanh = AdaptiveActivationFunction(torch.nn.Tanh()) - >>> AdaptiveTanh(torch.rand(3)) - tensor([0.1084, 0.3931, 0.7294], grad_fn=) - >>> AdaptiveTanh.alpha - Parameter containing: - tensor(1., requires_grad=True) - >>> - >>> # simple adaptive function with trainable parameters fixed alpha - >>> AdaptiveTanh = AdaptiveActivationFunction(torch.nn.Tanh(), - ... fixed=['alpha']) - >>> AdaptiveTanh.alpha - tensor(1.) - >>> AdaptiveTanh.beta - Parameter containing: - tensor(1., requires_grad=True) - >>> - .. seealso:: **Original reference**: Godfrey, Luke B., and Michael S. Gashler. @@ -51,14 +33,18 @@ class AdaptiveActivationFunction(torch.nn.Module): Vol. 1. IEEE, 2015. DOI: `arXiv preprint arXiv:1602.01321. `_. + Jagtap, Ameya D., Kenji Kawaguchi, and George Em Karniadakis. *Adaptive + activation functions accelerate convergence in deep and + physics-informed neural networks*. Journal of + Computational Physics 404 (2020): 109136. + DOI: `JCP 10.1016 + `_. """ - def __init__(self, func, alpha=None, beta=None, gamma=None, fixed=None): + def __init__(self, alpha=None, beta=None, gamma=None, fixed=None): """ - Initializes the AdaptiveActivationFunction module. + Initializes the Adaptive Function. - :param callable func: The original collable function. It could be an - initialized :meth:`torch.nn.Module`, or a python callable function. :param float | complex alpha: Scaling parameter alpha. Defaults to ``None``. When ``None`` is passed, the variable is initialized to 1. @@ -70,7 +56,7 @@ class AdaptiveActivationFunction(torch.nn.Module): the variable is initialized to 1. :param list fixed: List of parameters to fix during training, i.e. not optimized (``requires_grad`` set to ``False``). - Options are ['alpha', 'beta', 'gamma']. Defaults to None. + Options are ``alpha``, ``beta``, ``gamma``. Defaults to None. """ super().__init__() @@ -94,8 +80,6 @@ class AdaptiveActivationFunction(torch.nn.Module): check_consistency(alpha, (float, complex)) check_consistency(beta, (float, complex)) check_consistency(gamma, (float, complex)) - if not callable(func): - raise ValueError("Function must be a callable function.") # registering as tensors alpha = torch.tensor(alpha, requires_grad=False) @@ -119,34 +103,44 @@ class AdaptiveActivationFunction(torch.nn.Module): self._gamma = torch.nn.Parameter(gamma, requires_grad=True) else: self.register_buffer("gamma", gamma) - - # registering function - self._func = func + + # storing the activation + self._func = None def forward(self, x): """ - Forward pass of the function. - Applies the function to the input elementwise. + Define the computation performed at every call. + The function to the input elementwise. + + :param x: The input tensor to evaluate the activation function. + :type x: torch.Tensor | LabelTensor """ return self.alpha * (self._func(self.beta * x + self.gamma)) @property def alpha(self): """ - The alpha variable + The alpha variable. """ return self._alpha @property def beta(self): """ - The alpha variable + The beta variable. """ return self._beta @property def gamma(self): """ - The alpha variable + The gamma variable. """ return self._gamma + + @property + def func(self): + """ + The callable activation function. + """ + return self._func \ No newline at end of file diff --git a/pina/model/layers/__init__.py b/pina/model/layers/__init__.py index 5f5a14f..de9686b 100644 --- a/pina/model/layers/__init__.py +++ b/pina/model/layers/__init__.py @@ -12,7 +12,6 @@ __all__ = [ "PeriodicBoundaryEmbedding", "AVNOBlock", "LowRankBlock", - "AdaptiveActivationFunction", ] from .convolution_2d import ContinuousConvBlock @@ -26,5 +25,4 @@ from .fourier import FourierBlock1D, FourierBlock2D, FourierBlock3D from .pod import PODBlock from .embedding import PeriodicBoundaryEmbedding from .avno_layer import AVNOBlock -from .lowrank_layer import LowRankBlock -from .adaptive_func import AdaptiveActivationFunction +from .lowrank_layer import LowRankBlock \ No newline at end of file diff --git a/tests/test_adaptive_functions.py b/tests/test_adaptive_functions.py new file mode 100644 index 0000000..43d9c1b --- /dev/null +++ b/tests/test_adaptive_functions.py @@ -0,0 +1,62 @@ +import torch +import pytest + +from pina.adaptive_functions import (AdaptiveReLU, AdaptiveSigmoid, AdaptiveTanh, + AdaptiveSiLU, AdaptiveMish, AdaptiveELU, + AdaptiveCELU, AdaptiveGELU, AdaptiveSoftmin, + AdaptiveSoftmax, AdaptiveSIREN, AdaptiveExp) + + +adaptive_functions = (AdaptiveReLU, AdaptiveSigmoid, AdaptiveTanh, + AdaptiveSiLU, AdaptiveMish, AdaptiveELU, + AdaptiveCELU, AdaptiveGELU, AdaptiveSoftmin, + AdaptiveSoftmax, AdaptiveSIREN, AdaptiveExp) +x = torch.rand(10, requires_grad=True) + +@pytest.mark.parametrize("Func", adaptive_functions) +def test_constructor(Func): + if Func.__name__ == 'AdaptiveExp': + # simple + Func() + # setting values + af = Func(alpha=1., beta=2.) + assert af.alpha.requires_grad + assert af.beta.requires_grad + assert af.alpha == 1. + assert af.beta == 2. + else: + # simple + Func() + # setting values + af = Func(alpha=1., beta=2., gamma=3.) + assert af.alpha.requires_grad + assert af.beta.requires_grad + assert af.gamma.requires_grad + assert af.alpha == 1. + assert af.beta == 2. + assert af.gamma == 3. + + # fixed variables + af = Func(alpha=1., beta=2., fixed=['alpha']) + assert af.alpha.requires_grad is False + assert af.beta.requires_grad + assert af.alpha == 1. + assert af.beta == 2. + + with pytest.raises(TypeError): + Func(alpha=1., beta=2., fixed=['delta']) + + with pytest.raises(ValueError): + Func(alpha='s') + Func(alpha=1) + +@pytest.mark.parametrize("Func", adaptive_functions) +def test_forward(Func): + af = Func() + af(x) + +@pytest.mark.parametrize("Func", adaptive_functions) +def test_backward(Func): + af = Func() + y = af(x) + y.mean().backward() \ No newline at end of file diff --git a/tests/test_layers/test_adaptive_func.py b/tests/test_layers/test_adaptive_func.py deleted file mode 100644 index 3e3e666..0000000 --- a/tests/test_layers/test_adaptive_func.py +++ /dev/null @@ -1,48 +0,0 @@ -import torch -import pytest - -from pina.model.layers.adaptive_func import AdaptiveActivationFunction - -x = torch.rand(5) -torchfunc = torch.nn.Tanh() - -def test_constructor(): - # simple - AdaptiveActivationFunction(torchfunc) - - # setting values - af = AdaptiveActivationFunction(torchfunc, alpha=1., beta=2., gamma=3.) - assert af.alpha.requires_grad - assert af.beta.requires_grad - assert af.gamma.requires_grad - assert af.alpha == 1. - assert af.beta == 2. - assert af.gamma == 3. - - # fixed variables - af = AdaptiveActivationFunction(torchfunc, alpha=1., beta=2., - gamma=3., fixed=['alpha']) - assert af.alpha.requires_grad is False - assert af.beta.requires_grad - assert af.gamma.requires_grad - assert af.alpha == 1. - assert af.beta == 2. - assert af.gamma == 3. - - with pytest.raises(TypeError): - AdaptiveActivationFunction(torchfunc, alpha=1., beta=2., - gamma=3., fixed=['delta']) - - with pytest.raises(ValueError): - AdaptiveActivationFunction(torchfunc, alpha='s') - AdaptiveActivationFunction(torchfunc, alpha=1., fixed='alpha') - AdaptiveActivationFunction(torchfunc, alpha=1) - -def test_forward(): - af = AdaptiveActivationFunction(torchfunc) - af(x) - -def test_backward(): - af = AdaptiveActivationFunction(torchfunc) - y = af(x) - y.mean().backward() \ No newline at end of file