tenzr::nn::LayerNorm¶

Layer normalization module over the last dimension.

#include <norm.h>

Public Functions¶

	Name
	LayerNorm(int64_t normalized_dim, double eps, bool elementwise_affine, DType dtype)
template <typename T =float>	LayerNorm(int64_t normalized_dim, double eps =1.0e-5, bool elementwise_affine =true) Construct with the parameter dtype inferred from T.
template <typename T >	LayerNorm(std::type_identity< T > , int64_t normalized_dim, double eps =1.0e-5, bool elementwise_affine =true) Construct from an explicit std::type_identity dtype tag.
virtual Tensor	Forward(const Tensor & input) override
Tensor &	Weight()
const Tensor &	Weight() const
Tensor &	Bias()
const Tensor &	Bias() const
int64_t	NormalizedDim() const
double	Eps() const
virtual std::string	Name() const override
template <typename T =float> LayerNorm	Create(int64_t normalized_dim, double eps =1.0e-5, bool elementwise_affine =true) Static factory mirroring the templated constructor.

Additional inherited members¶

Public Functions inherited from tenzr::nn::Module

	Name
virtual	~Module() =default
Tensor	operator()(const Tensor & input)
virtual void	Train(bool mode)
void	Train() Switch to training mode.
void	Eval() Switch to evaluation mode.
bool	IsTraining() const
Tensor &	RegisterParameter(std::string name, Tensor param)
template <typename T ,typename... Args> std::shared_ptr< T >	RegisterSubmodule(std::string name, Args &&... args)
std::shared_ptr< Module >	RegisterSubmodule(std::string name, std::shared_ptr< Module > submod)
std::vector< Tensor >	Parameters(bool recurse =true) const
std::unordered_map< std::string, Tensor >	NamedParameters(std::string prefix ="", bool recurse =true) const
std::unordered_map< std::string, Tensor >	StateDict() const
void	LoadStateDict(const std::unordered_map< std::string, Tensor > & state)

Protected Attributes inherited from tenzr::nn::Module

	Name
std::unordered_map< std::string, Tensor >	parameters_
std::unordered_map< std::string, std::shared_ptr< Module > >	submodules_
bool	training_

Public Functions Documentation¶

function LayerNorm¶

LayerNorm(
    int64_t normalized_dim,
    double eps,
    bool elementwise_affine,
    DType dtype
)

Parameters:

normalized_dim Size of the normalized dimension.
eps Variance epsilon.
elementwise_affine Apply learnable weight and bias when true.
dtype Parameter dtype.
normalized_dim Size of the normalized dimension.
eps Stabilizing epsilon.
elementwise_affine Whether to allocate learnable scale and shift.
dtype Parameter dtype.

Construct with an explicit runtime dtype. Create a layer normalization over the last dimension.

function LayerNorm¶

template <typename T  =float>
inline explicit LayerNorm(
    int64_t normalized_dim,
    double eps =1.0e-5,
    bool elementwise_affine =true
)

Construct with the parameter dtype inferred from T.

function LayerNorm¶

template <typename T >
inline explicit LayerNorm(
    std::type_identity< T > ,
    int64_t normalized_dim,
    double eps =1.0e-5,
    bool elementwise_affine =true
)

Construct from an explicit std::type_identity dtype tag.

function Forward¶

virtual Tensor Forward(
    const Tensor & input
) override

Parameters:

input Input tensor.
input Input tensor whose last dimension equals normalized_dim.

Return:

Normalized tensor with the same shape.
Normalized tensor with the same shape.

Reimplements: tenzr::nn::Module::Forward

Normalize input over its last dimension. Normalize the last dimension by per-row mean and variance.

function Weight¶

inline Tensor & Weight()

Return: Mutable reference to the affine weight.

function Weight¶

inline const Tensor & Weight() const

Return: Const reference to the affine weight.

function Bias¶

inline Tensor & Bias()

Return: Mutable reference to the affine bias.

function Bias¶

inline const Tensor & Bias() const

Return: Const reference to the affine bias.

function NormalizedDim¶

inline int64_t NormalizedDim() const

Return: Size of the normalized dimension.

function Eps¶

inline double Eps() const

Return: Variance epsilon.

function Name¶

inline virtual std::string Name() const override

Return: "LayerNorm".

Reimplements: tenzr::nn::Module::Name

function Create¶

template <typename T  =float>
static inline LayerNorm Create(
    int64_t normalized_dim,
    double eps =1.0e-5,
    bool elementwise_affine =true
)

Static factory mirroring the templated constructor.