ó àÆ÷Xc@`s.ddlmZmZmZddlZddlZddlmZmZddlm Z ddl m Z ddl m Z mZddl mZddlmZdd lmZmZdd lmZmZmZmZd efd „ƒYZd d„Zd„Zdddddd„Zddd„Zdefd„ƒYZ defd„ƒYZ!defd„ƒYZ"e e gƒd„ƒZ#e e"gƒd„ƒZ$e e!gƒd„ƒZ%ej&j'j(ƒZ)de)_*ee)dd ƒe)j+d!e#d"dd ƒe)j+d#e$d"dd ƒe)j+d$e%d"dd ƒdS(%i(tabsolute_importtprint_functiontdivisionN(tApplytOp(tlocal_optimizer(tcopy_stack_trace(tas_tensor_variablet TensorType(tbasic(tregister_specialize_device(t Compositetas_common_dtype(taddtsubttrue_divtmult BNCompositecB`s5eZdZejjddƒd„ƒZd„ZRS(tdtypetcompute_test_valuetoffc C`sõ||_tjjd|ƒjƒ}tjjd|ƒjƒ}tjjd|ƒjƒ}tjjd|ƒjƒ}tjjd|ƒjƒ}tttt||ƒ|ƒ|ƒ|ƒ}|||||g}|g} t t |ƒj || ƒdS(NR( RttheanotscalartScalart make_variableR RRRtsuperRt__init__( tselfRtxtmeantstdtgammatbetatotinputstoutputs((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRs * cC`sw|\}}}}}|\}||} ||} | |} | } | |  ||} || |}| | | ||gS(N((RtinpstgradsRRRRR ttopt top_gammatx_meantdxtdmeantdstdtdgamma((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pytgrads    (sdtype(t__name__t __module__t init_paramRt configparsert change_flagsRR-(((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRs tlow_memcC`s€|dkrKtjjjdtd|jƒƒ}||||||ƒ}n1|dkrp|||||}n tdƒ‚|S(sM This function will build the symbolic graph for applying batch normalization to a set of activations. Also works on GPUs, but is not optimized using cuDNN. .. versionadded:: 0.7.1 Parameters ---------- inputs : symbolic tensor Mini-batch of activations gamma: symbolic tensor BN scale parameter, must be of same dimensionality as inputs and broadcastable against it beta: symbolic tensor BN shift parameter, must be of same dimensionality as inputs and broadcastable against it mean: symbolic tensor inputs means, must be of same dimensionality as inputs and broadcastable against it std: symbolic tensor inputs standard deviation, must be of same dimensionality as inputs and broadcastable against it mode: 'low_mem' or 'high_mem' Specify which batch_normalization implementation that will be used. As no intermediate representations are stored for the back-propagation, 'low_mem' implementation lower the memory usage, however, it is 5-10% slower than 'high_mem' implementation. Note that 5-10% computation time difference compare the batch_normalization operation only, time difference between implementation is likely to be less important on the full model fprop/bprop. R3t scalar_opRthigh_mems)mode must be either "low_mem", "high_mem"(RttensortelemwisetElemwiseRRt ValueError(R"RR RRtmodetelm_bntrval((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pytbatch_normalization*s" $  c`s(ˆdkrd ‰nqˆdkr=d ttd|ƒƒ‰nItˆtttjfƒrqtd„ˆDƒƒ‰ntdtˆƒƒ‚ttˆƒƒ‰t ˆƒdkr¹tdƒ‚nt ˆƒdksÝt ˆƒ|krütd|tˆƒfƒ‚nt‡fd †t|ƒDƒƒ}ˆ|fS( Nsper-activationitspatialics`s|]}t|ƒVqdS(N(tint(t.0ta((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pys ]ssinvalid axes: %ss/there should be at least one normalization axiss1axes should be less than ndim (<%d), but %s givenc3`s!|]}|ˆkr|VqdS(N((R@ti(taxes(s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pys es(i(i( ttupletranget isinstancetlisttnumpytndarrayR9tstrtsortedtlentmintmax(RCtndimt non_bc_axes((RCs5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyt!_prepare_batch_normalization_axesWs   $"sper-activationg-Cëâ6?gš™™™™™¹?c C`s•|j}t||ƒ\}} |j|kr6|} n@t| ƒ} dg|} x$t| ƒD]\} } | | | s%epsilon must be at least 1e-5, got %sRCtepsilontrunning_average_factort running_meant running_variiN(RORQRLt enumerateR9tNoneRHtcastRtconfigtfloatXRJRt dimshuffletTt addbroadcasttAbstractBatchNormTraint broadcastabletpatternbroadcastRD(R"RR RCRRRSRTRURORPt params_ndimtparams_dimshuffle_patternRBtaxist batchnorm_optoutRtinvstdtnew_running_meantnew_running_vartresultstr((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pytbatch_normalization_trainisjQ              *cC`s"|j}t||ƒ\}}|j|kr6|} n@t|ƒ} dg|} x$t|ƒD]\} } | | | s%epsilon must be at least 1e-5, got %sRCRR(RORQRLRVR9RHRXRRYRZRJRR[R\R]tAbstractBatchNormInference(R"RR RtvarRCRRRORPRaRbRBRcRd((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pytbatch_normalization_tests@4          R^cB`sYeZdZd Zd d„Zd„Zddd d d„Zd„Zd „Z d „Z RS(sÀ Abstract Op for Batch Normalization. Parameters ---------- axes : a tuple of ints The axes along which the input should be normalized. x : tensor The input to be normalized along `axes`. scale : tensor `scale` should have the same number of dimensions as `x`. All dimensions listed in `axes` should have length 1. bias : tensor `bias` should have the same number of dimensions as `x`. All dimensions listed in `axes` should have length 1. epsilon Epsilon value used in the batch normalization formula. Minimum allowed value is 1e-5 (imposed by cuDNN). running_average_factor : float Factor for updating the values or `running_mean` and `running_var`. If the factor is close to one, the running averages will update quickly, if the factor is close to zero it will update slowly. running_mean : tensor or None Previous value of the running mean. If this is given, the new value ``running_mean * (1 - running_average_factor) + batch mean * running_average_factor`` will be returned as one of the outputs of this function. `running_mean` and `running_var` should either both be given or both be None. running_var : tensor or None Previous value of the running variance. If this is given, the new value ``running_var * (1 - running_average_factor) + (m / (m - 1)) * batch var * running_average_factor`` will be returned as one of the outputs of this function, where `m` is the product of lengths of the averaged-over dimensions. `running_mean` and `running_var` should either both be given or both be None. RCicC`sVt|ttfƒst‚t|ƒdks3t‚td„|Dƒƒ}||_dS(Nics`s|]}t|ƒVqdS(N(R?(R@RA((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pys —s(RFRDRGtAssertionErrorRLRC(RRC((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyR”scC`s'|dg|dgt|jƒdS(Nii(RLR#(Rtnodetshape((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyt infer_shapešsg-Cëâ6?gš™™™™™¹?c C`st|ƒ}t|ƒ}t|ƒ}t|ƒ}t|ƒ}|dk rWt|ƒ}n|dk rrt|ƒ}n|j|jko’|jknst‚|dkrµ|dksÓ|dk rÍ|dk sÓt‚|dks÷|j|jks÷t‚|dks|j|jkst‚|rKt|||||ƒ\}}}}}nt|||ƒ\}}}|||||g}|jƒ|jƒ|jƒg} |dk r÷|dk r÷|j|ƒ|j|ƒ| j|jƒƒ| j|jƒƒnt||| ƒS(N(RRWRORoR ttypetappendR( RRtscaletbiasRRRSRTRUR"t output_types((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyt make_nodes4       +$$'!  cC`s¹|d \}}}}}|d} |d \} } } tjjƒƒtjjƒƒg} x6tdt|ƒƒD]}| jtjjƒƒƒqpWt|jƒ|| || | |ƒ| S(Niii(RtgradienttDisconnectedTypeRERLRttAbstractBatchNormTrainGradRC(RR"R#R%RRuRvRRRStdyt_R(tx_invstdtdisconnected_outputsRB((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pytL_op¾s  cC`s¿tttgtttgtttgtttgtttgg}xvtdt|jƒƒD]\}|djtƒx|dD]}|jtƒq}W|jtgd|dtgƒq[W|S(Niiii(tTruetFalseRERLR"Rt(RRptpatternsRBtpattern((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pytconnection_patternËs    'cC`sª|d \}}}}}|j} t| ƒdksIt| ƒ|jkrktd|jt| ƒfƒ‚n|j| dtƒ} |j| dtƒ} dt j | |ƒ} || || |} | |dd<| |dd<| |dd s(RFRDRGRoRLRC(RRC((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRscC`s |dgS(Ni((RRpRq((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRr sg-Cëâ6?cC`sát|ƒ}t|ƒ}t|ƒ}t|ƒ}t|ƒ}t|ƒ}t|||||ƒ\}}}}}|j|jko«|jko«|jko«|jkns¶t‚t|||||||g|jƒgƒS(N(RR RORoRRs(RRRuRvtestimated_meantestimated_varianceRR((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRxs      $Gc`s‚|\}}}}}}|d} |j‰tˆƒdksRtˆƒ|jkrttd|jtˆƒfƒ‚n‡fd†||||fDƒ\}}}}||} tjj| ƒ} tjj dƒ} | || } | ||j ˆdt ƒ| }| j ˆdt ƒ}| j ˆdt ƒ || }| ||j ˆdt ƒ || | | }| ||||tj j ƒƒgS(Nis1axes should be less than ndim (<%d), but %s givenc3`s$|]}tjj|ˆŒVqdS(N(RR6R](R@tt(RC(s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pys %sg@R†(RCRMRNROR9RJRR6R‡tconstanttsumRRyRz(RR"R%RRuRvtest_meantest_varRRR|t est_var_epstest_stdttwoR)tdscaletdbiasR*tdvar((RCs5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyR-s   '" " !.cC`s(tgtgtgtgtgtggS(N(RR‚(RRp((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyR…5sc C`sM|\}}}}}} |||tj|| ƒ|} | |ddEs(RFRDRGRoRLRC(RRC((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRBsg-Cëâ6?cC`sót|ƒ}t|ƒ}t|ƒ}t|ƒ}t|ƒ}t|ƒ}t|||||ƒ\}}}}}|j|jko«|jko«|jko«|jkns¶t‚t|||||||g|jƒ|jƒ|jƒgƒS(N(RR RORoRRs(RRR|RuR(R~RR((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRxHs      $GcC`s|d|d|dgS(Nii((RRpRq((s5/tmp/pip-build-X4mzal/theano/theano/tensor/nnet/bn.pyRrXscC`s<|\}}}}}} |j} t| ƒdksHt| ƒ|jkrjtd|jt| ƒfƒ‚n||} tj|| d| dtƒ} ||| | |d} || tj| d| dtƒ}tj ||| d| dtƒ}tj |d| dtƒ}||dd<||dd<||ddsF  " , d‹H/3#