Norms.hpp

Go to the documentation of this file.

// Copyright (c) 2019 Graphcore Ltd. All rights reserved.
#ifndef poplin_Norms_hpp
#define poplin_Norms_hpp
#include <functional>
#include <poplar/Graph.hpp>
#include <poplar/Program.hpp>
#include <tuple>
 
namespace poplin {
 
// Note that the functionality here should move to popnn once operations on
// broadcasting some of the dimensions is added. Currently these operations are
// optimised for tensors produced by convolutions/matrix multiplications.
// (see T6054)
 
poplar::Tensor createNormGamma(poplar::Graph &graph, const poplar::Tensor &acts,
                               const poplar::Type &type,
                               const poplar::DebugContext &debugContext = {});
 
poplar::Tensor createNormGamma(poplar::Graph &graph, const poplar::Tensor &acts,
                               const poplar::DebugContext &debugContext = {});
 
poplar::Tensor createNormBeta(poplar::Graph &graph, const poplar::Tensor &acts,
                              const poplar::Type &type,
                              const poplar::DebugContext &debugContext = {});
 
poplar::Tensor createNormBeta(poplar::Graph &graph, const poplar::Tensor &acts,
                              const poplar::DebugContext &debugContext = {});
 
std::pair<poplar::Tensor, poplar::Tensor>
createNormParams(poplar::Graph &graph, const poplar::Tensor &acts,
                 const poplar::DebugContext &debugContext = {});
 
//                        slower than when set to false.
std::pair<poplar::Tensor, poplar::Tensor>
normStatistics(poplar::Graph &graph, const poplar::Tensor &actsUngrouped,
               float eps, poplar::program::Sequence &prog,
               bool unbiasedVarEstimate, bool stableAlgo = false,
               const poplar::Type &partialsType = poplar::FLOAT,
               const poplar::DebugContext &debugContext = {});
 
using DistributedNormReduceCallback = std::function<std::vector<poplar::Tensor>(
    poplar::Graph &replicatedGraph,
    const std::vector<poplar::Tensor> &inputsToReduce,
    poplar::program::Sequence &prog, unsigned groupSize,
    const poplar::DebugContext &debugContext,
    const poplar::OptionFlags &options)>;
 
//                        slower than when set to false.
std::pair<poplar::Tensor, poplar::Tensor> distributedNormStatistics(
    poplar::Graph &replicatedGraph, const poplar::Tensor &actsUngrouped,
    float eps, poplar::program::Sequence &prog, bool unbiasedVarEstimate,
    DistributedNormReduceCallback allReduceCallback, unsigned normSize,
    bool stableAlgo = false, const poplar::Type &partialsType = poplar::FLOAT,
    const poplar::DebugContext &debugContext = {});
 
poplar::Tensor normWhiten(poplar::Graph &graph, const poplar::Tensor &acts,
                          const poplar::Tensor &mean,
                          const poplar::Tensor &iStdDev,
                          poplar::program::Sequence &prog,
                          const poplar::DebugContext &debugContext = {});
 
poplar::Tensor normalise(poplar::Graph &graph,
                         const poplar::Tensor &actsWhitened,
                         const poplar::Tensor &gamma,
                         const poplar::Tensor &beta,
                         poplar::program::Sequence &prog,
                         const poplar::DebugContext &debugContext = {});
 
std::pair<poplar::Tensor, poplar::Tensor>
normParamGradients(poplar::Graph &graph, const poplar::Tensor &actsWhitened,
                   const poplar::Tensor &gradsIn,
                   poplar::program::Sequence &prog,
                   const poplar::Type &partialsType = poplar::FLOAT,
                   const poplar::DebugContext &debugContext = {});
 
poplar::Tensor normGradients(poplar::Graph &graph,
                             const poplar::Tensor &gradsIn,
                             const poplar::Tensor &gamma,
                             poplar::program::Sequence &prog,
                             const poplar::DebugContext &debugContext = {});
 
poplar::Tensor normStatisticsGradients(
    poplar::Graph &graph, const poplar::Tensor &actsWhitened,
    const poplar::Tensor &gradsIn, const poplar::Tensor &invStdDev,
    poplar::program::Sequence &prog,
    const poplar::Type &partialsType = poplar::FLOAT,
    const poplar::DebugContext &debugContext = {});
 
poplar::Tensor distributedNormStatisticsGradients(
    poplar::Graph &replicatedGraph, const poplar::Tensor &actsWhitened,
    const poplar::Tensor &gradsIn, const poplar::Tensor &invStdDev,
    poplar::program::Sequence &prog,
    poplin::DistributedNormReduceCallback reduceCallback, unsigned normSize,
    const poplar::Type &partialsType = poplar::FLOAT,
    const poplar::DebugContext &debugContext = {});
 
} // namespace poplin
 
#endif // poplin_Norms_hpp