# megengine.module.activation 源代码

# -*- coding: utf-8 -*-
#
#
# Unless required by applicable law or agreed to in writing,
# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
import numpy as np

from ..functional import leaky_relu, prelu, relu, sigmoid, softmax
from ..tensor import Parameter
from .module import Module

[文档]class Softmax(Module):
r"""
Applies a softmax function. Softmax is defined as:

.. math::
\text{Softmax}(x_{i}) = \frac{exp(x_i)}{\sum_j exp(x_j)}

It is applied to all elements along axis, and rescales elements so that
they stay in the range [0, 1] and sum to 1.

:param axis: Along which axis softmax will be applied. By default,
softmax will apply along the highest ranked axis.

Examples:

.. testcode::

import numpy as np
import megengine as mge
import megengine.module as M

data = mge.tensor(np.array([-2,-1,0,1,2]).astype(np.float32))
softmax = M.Softmax()
output = softmax(data)
with np.printoptions(precision=6):
print(output.numpy())

Outputs:

.. testoutput::

[0.011656 0.031685 0.086129 0.234122 0.636409]

"""

def __init__(self, axis=None):
super().__init__()
self.axis = axis

[文档]    def forward(self, inputs):
return softmax(inputs, self.axis)

def _module_info_string(self) -> str:
return "axis={axis}".format(axis=self.axis)

[文档]class Sigmoid(Module):
r"""
Applies the element-wise function:

.. math::
\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}

Examples:

.. testcode::

import numpy as np
import megengine as mge
import megengine.module as M

data = mge.tensor(np.array([-2,-1,0,1,2,]).astype(np.float32))
sigmoid = M.Sigmoid()
output = sigmoid(data)
with np.printoptions(precision=6):
print(output.numpy())

Outputs:

.. testoutput::

[0.119203 0.268941 0.5      0.731059 0.880797]

"""

[文档]    def forward(self, inputs):
return sigmoid(inputs)

[文档]class ReLU(Module):
r"""
Applies the element-wise function:

.. math::
\text{ReLU}(x) = \max(x, 0)

Examples:

.. testcode::

import numpy as np
import megengine as mge
import megengine.module as M
data = mge.tensor(np.array([-2,-1,0,1,2,]).astype(np.float32))
relu = M.ReLU()
output = relu(data)
with np.printoptions(precision=6):
print(output.numpy())

Outputs:

.. testoutput::

[0. 0. 0. 1. 2.]

"""

[文档]    def forward(self, x):
return relu(x)

[文档]class PReLU(Module):
r"""
Applies the element-wise function:

.. math::
\text{PReLU}(x) = \max(0,x) + a * \min(0,x)

or

.. math::
\text{PReLU}(x) =
\begin{cases}
x, & \text{ if } x \geq 0 \\
ax, & \text{ otherwise }
\end{cases}

Here :math:a is a learnable parameter. When called without arguments, PReLU() uses
a single paramter :math:a across all input channel. If called with PReLU(num_of_channels), each input channle will has it's own :math:a.

:param num_parameters: number of :math:a to learn, there is only two
values are legitimate: 1, or the number of channels at input. Default: 1
:param init: the initial value of :math:a. Default: 0.25

Examples:

.. testcode::

import numpy as np
import megengine as mge
import megengine.module as M
data = mge.tensor(np.array([-1.2, -3.7, 2.7]).astype(np.float32))
prelu = M.PReLU()
output = prelu(data)
print(output.numpy())

Outputs:

.. testoutput::

[-0.3   -0.925  2.7  ]

"""

def __init__(self, num_parameters: int = 1, init: float = 0.25):
super().__init__()
self.num_parameters = num_parameters
if num_parameters > 1:
# Assume format is NCHW
self.weight = Parameter(
data=np.full((1, num_parameters, 1, 1), init, dtype=np.float32)
)
else:
self.weight = Parameter(data=[init])

[文档]    def forward(self, inputs):
assert self.weight.shape == (1,) or self.weight.shape == (
1,
int(inputs.shape[1]),
1,
1,
), "invalid weight's shape"
return prelu(inputs, self.weight)

[文档]class LeakyReLU(Module):
r"""
Applies the element-wise function:

.. math::
\text{LeakyReLU}(x) = \max(0,x) + negative\_slope \times \min(0,x)

or

.. math::
\text{LeakyReLU}(x) =
\begin{cases}
x, & \text{ if } x \geq 0 \\
negative\_slope \times x, & \text{ otherwise }
\end{cases}

Examples:

.. testcode::

import numpy as np
import megengine as mge
import megengine.module as M
data = mge.tensor(np.array([-8, -12, 6, 10]).astype(np.float32))

leakyrelu = M.LeakyReLU(0.01)
output = leakyrelu(data)
print(output.numpy())

Outputs:

.. testoutput::

[-0.08 -0.12  6.   10.  ]

"""

def __init__(self, negative_slope: float = 0.01):
super().__init__()
self.negative_slope = negative_slope

[文档]    def forward(self, inputs):
return leaky_relu(inputs, self.negative_slope)