| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | module athena__initialiser_lecun | ||
| 2 | !! Module containing the implementation of the LeCun initialiser | ||
| 3 | !! | ||
| 4 | !! This module implements LeCun initialisation, the precursor to modern | ||
| 5 | !! initialisation schemes, designed for efficient backpropagation. | ||
| 6 | !! | ||
| 7 | !! Mathematical operation: | ||
| 8 | !! Uniform variant: W ~ U(-limit, limit) | ||
| 9 | !! where limit = sqrt(3 / fan_in) | ||
| 10 | !! Normal variant: W ~ N(0, σ²) | ||
| 11 | !! where σ = sqrt(1 / fan_in) | ||
| 12 | !! | ||
| 13 | !! fan_in is the number of input units to the layer. | ||
| 14 | !! | ||
| 15 | !! Rationale: Maintains variance of inputs through layers | ||
| 16 | !! Helps prevent saturation of activation functions | ||
| 17 | !! | ||
| 18 | !! Best for: SELU activation (with lecun_normal variant) | ||
| 19 | !! Also works with: Tanh, Sigmoid | ||
| 20 | !! Reference: LeCun et al. (1998), Neural Networks: Tricks of the Trade | ||
| 21 | use coreutils, only: real32, pi, stop_program | ||
| 22 | use athena__misc_types, only: base_init_type | ||
| 23 | implicit none | ||
| 24 | |||
| 25 | |||
| 26 | private | ||
| 27 | |||
| 28 | public :: lecun_uniform_init_type | ||
| 29 | public :: lecun_normal_init_type | ||
| 30 | |||
| 31 | |||
| 32 | type, extends(base_init_type) :: lecun_uniform_init_type | ||
| 33 | !! Type for the LeCun initialiser (uniform) | ||
| 34 | contains | ||
| 35 | procedure, pass(this) :: initialise => lecun_uniform_initialise | ||
| 36 | !! Initialise the weights and biases using the LeCun uniform distribution | ||
| 37 | end type lecun_uniform_init_type | ||
| 38 | type, extends(base_init_type) :: lecun_normal_init_type | ||
| 39 | !! Type for the LeCun initialiser (normal) | ||
| 40 | contains | ||
| 41 | procedure, pass(this) :: initialise => lecun_normal_initialise | ||
| 42 | !! Initialise the weights and biases using the LeCun normal distribution | ||
| 43 | end type lecun_normal_init_type | ||
| 44 | |||
| 45 | |||
| 46 | interface lecun_uniform_init_type | ||
| 47 | module function initialiser_lecun_uniform_setup() result(initialiser) | ||
| 48 | !! Interface for the LeCun uniform initialiser | ||
| 49 | type(lecun_uniform_init_type) :: initialiser | ||
| 50 | !! LeCun uniform initialiser object | ||
| 51 | end function initialiser_lecun_uniform_setup | ||
| 52 | end interface lecun_uniform_init_type | ||
| 53 | |||
| 54 | interface lecun_normal_init_type | ||
| 55 | module function initialiser_lecun_normal_setup() result(initialiser) | ||
| 56 | !! Interface for the LeCun normal initialiser | ||
| 57 | type(lecun_normal_init_type) :: initialiser | ||
| 58 | !! LeCun normal initialiser object | ||
| 59 | end function initialiser_lecun_normal_setup | ||
| 60 | end interface lecun_normal_init_type | ||
| 61 | |||
| 62 | |||
| 63 | |||
| 64 | contains | ||
| 65 | |||
| 66 | !############################################################################### | ||
| 67 | 4 | module function initialiser_lecun_uniform_setup() result(initialiser) | |
| 68 | !! Interface for the LeCun uniform initialiser | ||
| 69 | implicit none | ||
| 70 | |||
| 71 | type(lecun_uniform_init_type) :: initialiser | ||
| 72 | !! LeCun uniform initialiser object | ||
| 73 | |||
| 74 | 4 | initialiser%name = "lecun_uniform" | |
| 75 | |||
| 76 | 4 | end function initialiser_lecun_uniform_setup | |
| 77 | !------------------------------------------------------------------------------- | ||
| 78 | 4 | module function initialiser_lecun_normal_setup() result(initialiser) | |
| 79 | !! Interface for the LeCun normal initialiser | ||
| 80 | implicit none | ||
| 81 | |||
| 82 | type(lecun_normal_init_type) :: initialiser | ||
| 83 | !! LeCun normal initialiser object | ||
| 84 | |||
| 85 | 4 | initialiser%name = "lecun_normal" | |
| 86 | |||
| 87 | 4 | end function initialiser_lecun_normal_setup | |
| 88 | !############################################################################### | ||
| 89 | |||
| 90 | |||
| 91 | !############################################################################### | ||
| 92 | 7 | subroutine lecun_uniform_initialise(this, input, fan_in, fan_out, spacing) | |
| 93 | !! Initialise the weights and biases using the LeCun uniform distribution | ||
| 94 | implicit none | ||
| 95 | |||
| 96 | ! Arguments | ||
| 97 | class(lecun_uniform_init_type), intent(inout) :: this | ||
| 98 | !! Instance of the Glorot initialiser | ||
| 99 | real(real32), dimension(..), intent(out) :: input | ||
| 100 | !! Weights and biases to initialise | ||
| 101 | integer, optional, intent(in) :: fan_in, fan_out | ||
| 102 | !! Number of input and output units | ||
| 103 | integer, dimension(:), optional, intent(in) :: spacing | ||
| 104 | !! Spacing of the input and output units (not used) | ||
| 105 | |||
| 106 | ! Local variables | ||
| 107 | integer :: n | ||
| 108 | !! Number of elements in the input array | ||
| 109 | real(real32) :: limit | ||
| 110 | !! Scaling factor | ||
| 111 | 7 | real(real32), dimension(:), allocatable :: r | |
| 112 | !! Temporary uniform random numbers | ||
| 113 | |||
| 114 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | if(.not.present(fan_in)) & |
| 115 | ✗ | call stop_program("lecun_uniform_initialise: fan_in not present") | |
| 116 | |||
| 117 | 7 | limit = sqrt(3._real32 / real(fan_in, real32)) | |
| 118 |
4/6✗ Branch 0 not taken.
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 13 times.
✓ Branch 4 taken 13 times.
✓ Branch 5 taken 7 times.
|
20 | n = size(input) |
| 119 |
7/14✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 7 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 7 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 7 times.
|
7 | allocate(r(n)) |
| 120 | 7 | call random_number(r) | |
| 121 |
6/10✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 7 times.
✓ Branch 12 taken 3470 times.
✓ Branch 13 taken 7 times.
|
3477 | r = (2._real32 * r - 1._real32) * limit |
| 122 | |||
| 123 | ! Assign according to rank | ||
| 124 | select rank(input) | ||
| 125 | rank(0) | ||
| 126 |
2/4✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
|
1 | input = r(1) |
| 127 | rank(1) | ||
| 128 |
11/20✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 4 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 4 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 4 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 4 times.
✗ Branch 15 not taken.
✓ Branch 16 taken 4 times.
✗ Branch 18 not taken.
✓ Branch 19 taken 4 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 4 times.
✗ Branch 24 not taken.
✓ Branch 25 taken 4 times.
✓ Branch 27 taken 3467 times.
✓ Branch 28 taken 4 times.
|
3471 | input = r |
| 129 | rank(2) | ||
| 130 | ✗ | input = reshape(r, shape(input)) | |
| 131 | rank(3) | ||
| 132 |
4/6✓ Branch 0 taken 3 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
|
4 | input = reshape(r, shape(input)) |
| 133 | rank(4) | ||
| 134 | ✗ | input = reshape(r, shape(input)) | |
| 135 | rank(5) | ||
| 136 | ✗ | input = reshape(r, shape(input)) | |
| 137 | rank(6) | ||
| 138 |
4/6✓ Branch 0 taken 6 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 6 times.
|
7 | input = reshape(r, shape(input)) |
| 139 | end select | ||
| 140 | |||
| 141 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | deallocate(r) |
| 142 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | end subroutine lecun_uniform_initialise |
| 143 | !############################################################################### | ||
| 144 | |||
| 145 | |||
| 146 | !############################################################################### | ||
| 147 | 7 | subroutine lecun_normal_initialise(this, input, fan_in, fan_out, spacing) | |
| 148 | !! Initialise the weights and biases using the LeCun normal distribution | ||
| 149 | implicit none | ||
| 150 | |||
| 151 | ! Arguments | ||
| 152 | class(lecun_normal_init_type), intent(inout) :: this | ||
| 153 | !! Instance of the LeCun initialiser | ||
| 154 | real(real32), dimension(..), intent(out) :: input | ||
| 155 | !! Weights and biases to initialise | ||
| 156 | integer, optional, intent(in) :: fan_in, fan_out | ||
| 157 | !! Number of input and output parameters | ||
| 158 | integer, dimension(:), optional, intent(in) :: spacing | ||
| 159 | !! Spacing of the input and output units (not used) | ||
| 160 | |||
| 161 | ! Local variables | ||
| 162 | integer :: n | ||
| 163 | !! Number of elements in the input array | ||
| 164 | real(real32) :: sigma | ||
| 165 | !! Scaling factor | ||
| 166 | 7 | real(real32), dimension(:), allocatable :: u1, u2, z | |
| 167 | !! Temporary arrays for the random numbers | ||
| 168 | |||
| 169 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | if(.not.present(fan_in)) & |
| 170 | ✗ | call stop_program("lecun_normal_initialise: fan_in not present") | |
| 171 | |||
| 172 | 7 | sigma = sqrt(1._real32/real(fan_in,real32)) | |
| 173 |
4/6✗ Branch 0 not taken.
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 13 times.
✓ Branch 4 taken 13 times.
✓ Branch 5 taken 7 times.
|
20 | n = size(input) |
| 174 |
21/42✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 7 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 7 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 7 times.
✓ Branch 17 taken 7 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✓ Branch 20 taken 7 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 7 times.
✗ Branch 23 not taken.
✓ Branch 24 taken 7 times.
✗ Branch 25 not taken.
✓ Branch 26 taken 7 times.
✗ Branch 28 not taken.
✓ Branch 29 taken 7 times.
✗ Branch 31 not taken.
✓ Branch 32 taken 7 times.
✓ Branch 34 taken 7 times.
✗ Branch 35 not taken.
✗ Branch 36 not taken.
✓ Branch 37 taken 7 times.
✗ Branch 38 not taken.
✓ Branch 39 taken 7 times.
✗ Branch 40 not taken.
✓ Branch 41 taken 7 times.
✗ Branch 42 not taken.
✓ Branch 43 taken 7 times.
✗ Branch 45 not taken.
✓ Branch 46 taken 7 times.
✗ Branch 48 not taken.
✓ Branch 49 taken 7 times.
|
7 | allocate(u1(n), u2(n), z(n)) |
| 175 | |||
| 176 | 7 | call random_number(u1) | |
| 177 | 7 | call random_number(u2) | |
| 178 |
8/14✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 7 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 7 times.
✓ Branch 15 taken 3470 times.
✓ Branch 16 taken 7 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 3470 times.
|
3477 | where (u1 .lt. 1.E-7_real32) |
| 179 |
4/8✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 7 times.
|
7 | u1 = 1.E-7_real32 |
| 180 | end where | ||
| 181 | |||
| 182 | ! Box-Muller transform | ||
| 183 |
13/26✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 7 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 7 times.
✗ Branch 15 not taken.
✓ Branch 16 taken 7 times.
✗ Branch 18 not taken.
✓ Branch 19 taken 7 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 7 times.
✗ Branch 24 not taken.
✓ Branch 25 taken 7 times.
✓ Branch 27 taken 7 times.
✗ Branch 28 not taken.
✓ Branch 29 taken 7 times.
✗ Branch 30 not taken.
✗ Branch 31 not taken.
✗ Branch 32 not taken.
✓ Branch 33 taken 3470 times.
✓ Branch 34 taken 7 times.
|
3477 | z = sqrt(-2._real32 * log(u1)) * cos(2._real32 * pi * u2) |
| 184 |
6/10✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 7 times.
✓ Branch 12 taken 3470 times.
✓ Branch 13 taken 7 times.
|
3477 | z = sigma * z |
| 185 | |||
| 186 | select rank(input) | ||
| 187 | rank(0) | ||
| 188 |
2/4✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
|
1 | input = z(1) |
| 189 | rank(1) | ||
| 190 |
11/20✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 4 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 4 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 4 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 4 times.
✗ Branch 15 not taken.
✓ Branch 16 taken 4 times.
✗ Branch 18 not taken.
✓ Branch 19 taken 4 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 4 times.
✗ Branch 24 not taken.
✓ Branch 25 taken 4 times.
✓ Branch 27 taken 3467 times.
✓ Branch 28 taken 4 times.
|
3471 | input = z |
| 191 | rank(2) | ||
| 192 | ✗ | input = reshape(z, shape(input)) | |
| 193 | rank(3) | ||
| 194 |
4/6✓ Branch 0 taken 3 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 3 times.
|
4 | input = reshape(z, shape(input)) |
| 195 | rank(4) | ||
| 196 | ✗ | input = reshape(z, shape(input)) | |
| 197 | rank(5) | ||
| 198 | ✗ | input = reshape(z, shape(input)) | |
| 199 | rank(6) | ||
| 200 |
4/6✓ Branch 0 taken 6 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 6 times.
|
7 | input = reshape(z, shape(input)) |
| 201 | end select | ||
| 202 | |||
| 203 |
3/6✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 7 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 7 times.
|
7 | deallocate(u1, u2, z) |
| 204 | |||
| 205 |
3/6✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 7 times.
|
7 | end subroutine lecun_normal_initialise |
| 206 | !############################################################################### | ||
| 207 | |||
| 208 |
26/36✓ Branch 0 taken 6 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 6 times.
✓ Branch 6 taken 4 times.
✓ Branch 7 taken 2 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 2 times.
✓ Branch 10 taken 1 times.
✓ Branch 11 taken 1 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 1 times.
✗ Branch 14 not taken.
✓ Branch 15 taken 1 times.
✓ Branch 16 taken 1 times.
✗ Branch 17 not taken.
✓ Branch 18 taken 6 times.
✓ Branch 19 taken 1 times.
✓ Branch 20 taken 6 times.
✗ Branch 21 not taken.
✓ Branch 22 taken 1 times.
✓ Branch 23 taken 6 times.
✓ Branch 24 taken 4 times.
✓ Branch 25 taken 2 times.
✗ Branch 26 not taken.
✓ Branch 27 taken 2 times.
✓ Branch 28 taken 1 times.
✓ Branch 29 taken 1 times.
✗ Branch 30 not taken.
✓ Branch 31 taken 1 times.
✗ Branch 32 not taken.
✓ Branch 33 taken 1 times.
✓ Branch 34 taken 1 times.
✗ Branch 35 not taken.
|
70 | end module athena__initialiser_lecun |
| 209 |