Poplar and PopLibs
NonLinearityDef.hpp
Go to the documentation of this file.
1// Copyright (c) 2016 Graphcore Ltd. All rights reserved.
6#ifndef popnn_NonLinearityDef_hpp
7#define popnn_NonLinearityDef_hpp
8
9namespace popnn {
10
11enum class NonLinearityType {
14 SIGMOID,
17 HARD_SIGMOID,
21 RELU,
24 TANH,
30 GELU,
31 // SWISH:
32 // * y = x / (1 + e^(-x))
33 SWISH,
37 SOFTMAX,
39 SOFTMAX_STABLE,
42 SOFTMAX_SCALED
43};
44
45} // end namespace popnn
46
47#endif // popnn_NonLinearityDef_hpp
popnn
Functions used in neural networks.
Definition: BatchNorm.hpp:14
popnn::NonLinearityType
NonLinearityType
Definition: NonLinearityDef.hpp:11
popnn::NonLinearityType::TANH
@ TANH
Hyperbolic tangent:
popnn::NonLinearityType::SOFTMAX_STABLE
@ SOFTMAX_STABLE
Same as SOFTMAX, but slower more numerically stable algorithm used.
popnn::NonLinearityType::HARD_SIGMOID
@ HARD_SIGMOID
Hard Sigmoid:
popnn::NonLinearityType::GELU
@ GELU
Gaussian Error Linear Unit:
popnn::NonLinearityType::SOFTMAX_SCALED
@ SOFTMAX_SCALED
Same as SOFTMAX, but slower more numerically stable algorithm used.
popnn::NonLinearityType::RELU
@ RELU
Rectified Linear Unit: