Locally connected layer (#201)

* changing reshape layer

* added tests; to note that they don't work

* Now reshape2d works, maxpool still not

* Saving changes before rebasing

* Resolved merge conflicts

* Bug fixed; Added conv1d; Conv1d and maxpool backward still not working

* Bug fixes; now everything works

* Updated the comments

* Implemented locally connected 1d

* Bug fix

* Bug fix

* New bugs

* Bug fix

* Definitive bug fixes

* Adding jvdp1's review

* Implemented OneAdder's suggestions

* Deleting useless variables

* again

* Update src/nf/nf_conv1d_layer_submodule.f90

Co-authored-by: Jeremie Vandenplas <jeremie.vandenplas@gmail.com>

* locally_connected_1d -> locally_connected1d

* Update features table

* Fix CMakeLists

* Fix CmakeListst

* Another one

* Tidy up

* Acknowledge contributors

---------

Co-authored-by: Jeremie Vandenplas <jeremie.vandenplas@gmail.com>
Co-authored-by: milancurcic <caomaco@gmail.com>

Author: 3 people

CMakeLists.txt

@@ -18,6 +18,8 @@ add_library(neural-fortran
   src/nf.f90
   src/nf/nf_activation.f90
   src/nf/nf_base_layer.f90
+  src/nf/nf_conv1d_layer.f90
+  src/nf/nf_conv1d_layer_submodule.f90
   src/nf/nf_conv2d_layer.f90
   src/nf/nf_conv2d_layer_submodule.f90
   src/nf/nf_cross_attention_layer.f90
@@ -41,12 +43,16 @@ add_library(neural-fortran
   src/nf/nf_layernorm_submodule.f90
   src/nf/nf_layer.f90
   src/nf/nf_layer_submodule.f90
+  src/nf/nf_locally_connected1d_layer_submodule.f90
+  src/nf/nf_locally_connected1d_layer.f90
   src/nf/nf_linear2d_layer.f90
   src/nf/nf_linear2d_layer_submodule.f90
   src/nf/nf_embedding_layer.f90
   src/nf/nf_embedding_layer_submodule.f90
   src/nf/nf_loss.f90
   src/nf/nf_loss_submodule.f90
+  src/nf/nf_maxpool1d_layer.f90
+  src/nf/nf_maxpool1d_layer_submodule.f90
   src/nf/nf_maxpool2d_layer.f90
   src/nf/nf_maxpool2d_layer_submodule.f90
   src/nf/nf_metrics.f90
@@ -60,6 +66,8 @@ add_library(neural-fortran
   src/nf/nf_random.f90
   src/nf/nf_reshape_layer.f90
   src/nf/nf_reshape_layer_submodule.f90
+  src/nf/nf_reshape2d_layer.f90
+  src/nf/nf_reshape2d_layer_submodule.f90
   src/nf/nf_self_attention_layer.f90
   src/nf/io/nf_io_binary.f90
   src/nf/io/nf_io_binary_submodule.f90

README.md

@@ -33,16 +33,18 @@ Read the paper [here](https://arxiv.org/abs/1902.06714).
 | Embedding | `embedding` | n/a | 2 | ✅ | ✅ |
 | Dense (fully-connected) | `dense` | `input1d`, `dense`, `dropout`, `flatten` | 1 | ✅ | ✅ |
 | Dropout | `dropout` | `dense`, `flatten`, `input1d` | 1 | ✅ | ✅ |
-| Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅(*) |
+| Locally connected (1-d) | `locally_connected1d` | `input2d`, `locally_connected1d`, `conv1d`, `maxpool1d`, `reshape2d` | 2 | ✅ | ✅ |
+| Convolutional (1-d) | `conv1d` | `input2d`, `conv1d`, `maxpool1d`, `reshape2d` | 2 | ✅ | ✅ |
+| Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅ |
+| Max-pooling (1-d) | `maxpool1d` | `input2d`, `conv1d`, `maxpool1d`, `reshape2d` | 2 | ✅ | ✅ |
 | Max-pooling (2-d) | `maxpool2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅ |
 | Linear (2-d) | `linear2d` | `input2d`, `layernorm`, `linear2d`, `self_attention` | 2 | ✅ | ✅ |
 | Self-attention | `self_attention` | `input2d`, `layernorm`, `linear2d`, `self_attention` | 2 | ✅ | ✅ |
 | Layer Normalization | `layernorm` | `linear2d`, `self_attention` | 2 | ✅ | ✅ |
 | Flatten | `flatten` | `input2d`, `input3d`, `conv2d`, `maxpool2d`, `reshape` | 1 | ✅ | ✅ |
+| Reshape (1-d to 2-d) | `reshape2d` | `input2d`, `conv1d`, `locally_connected1d`, `maxpool1d` | 2 | ✅ | ✅ |
 | Reshape (1-d to 3-d) | `reshape` | `input1d`, `dense`, `flatten` | 3 | ✅ | ✅ |
 
-(*) See Issue [#145](https://github.com/modern-fortran/neural-fortran/issues/145) regarding non-converging CNN training on the MNIST dataset.
-
 ## Getting started
 
 Get the code:
@@ -267,7 +269,9 @@ Thanks to all open-source contributors to neural-fortran:
 [jvdp1](https://github.com/jvdp1),
 [jvo203](https://github.com/jvo203),
 [milancurcic](https://github.com/milancurcic),
+[OneAdder](https://github.com/OneAdder),
 [pirpyn](https://github.com/pirpyn),
+[rico07](https://github.com/ricor07),
 [rouson](https://github.com/rouson),
 [rweed](https://github.com/rweed),
 [Spnetic-5](https://github.com/Spnetic-5),

example/CMakeLists.txt

@@ -1,5 +1,6 @@
 foreach(execid
   cnn_mnist
+  cnn_mnist_1d
   dense_mnist
   get_set_network_params
   network_parameters

example/cnn_mnist.f90

@@ -12,7 +12,7 @@ program cnn_mnist
   real, allocatable :: validation_images(:,:), validation_labels(:)
   real, allocatable :: testing_images(:,:), testing_labels(:)
   integer :: n
-  integer, parameter :: num_epochs = 10
+  integer, parameter :: num_epochs = 250
 
   call load_mnist(training_images, training_labels, &
                   validation_images, validation_labels, &
@@ -35,9 +35,9 @@ program cnn_mnist
     call net % train( &
       training_images, &
       label_digits(training_labels), &
-      batch_size=128, &
+      batch_size=16, &
       epochs=1, &
-      optimizer=sgd(learning_rate=3.) &
+      optimizer=sgd(learning_rate=0.001) &
     )
 
     print '(a,i2,a,f5.2,a)', 'Epoch ', n, ' done, Accuracy: ', accuracy( &

example/cnn_mnist_1d.f90

@@ -0,0 +1,67 @@
+program cnn_mnist_1d
+
+    use nf, only: network, sgd, &
+      input, conv1d, maxpool1d, flatten, dense, reshape, reshape2d, locally_connected1d, &
+      load_mnist, label_digits, softmax, relu
+  
+    implicit none
+  
+    type(network) :: net
+  
+    real, allocatable :: training_images(:,:), training_labels(:)
+    real, allocatable :: validation_images(:,:), validation_labels(:)
+    real, allocatable :: testing_images(:,:), testing_labels(:)
+    integer :: n
+    integer, parameter :: num_epochs = 250
+  
+    call load_mnist(training_images, training_labels, &
+                    validation_images, validation_labels, &
+                    testing_images, testing_labels)
+  
+    net = network([ &
+      input(784), &
+      reshape2d([28, 28]), &
+      locally_connected1d(filters=8, kernel_size=3, activation=relu()), &
+      maxpool1d(pool_size=2), &
+      locally_connected1d(filters=16, kernel_size=3, activation=relu()), &
+      maxpool1d(pool_size=2), &
+      dense(10, activation=softmax()) &
+    ])
+  
+    call net % print_info()
+  
+    epochs: do n = 1, num_epochs
+  
+      call net % train( &
+        training_images, &
+        label_digits(training_labels), &
+        batch_size=16, &
+        epochs=1, &
+        optimizer=sgd(learning_rate=0.01) &
+      )
+  
+      print '(a,i2,a,f5.2,a)', 'Epoch ', n, ' done, Accuracy: ', accuracy( &
+        net, validation_images, label_digits(validation_labels)) * 100, ' %'
+  
+    end do epochs
+  
+    print '(a,f5.2,a)', 'Testing accuracy: ', &
+      accuracy(net, testing_images, label_digits(testing_labels)) * 100, '%'
+  
+  contains
+  
+    real function accuracy(net, x, y)
+      type(network), intent(in out) :: net
+      real, intent(in) :: x(:,:), y(:,:)
+      integer :: i, good
+      good = 0
+      do i = 1, size(x, dim=2)
+        if (all(maxloc(net % predict(x(:,i))) == maxloc(y(:,i)))) then
+          good = good + 1
+        end if
+      end do
+      accuracy = real(good) / size(x, dim=2)
+    end function accuracy
+  
+  end program cnn_mnist_1d
+  

src/nf.f90

@@ -3,6 +3,7 @@ module nf
   use nf_datasets_mnist, only: label_digits, load_mnist
   use nf_layer, only: layer
   use nf_layer_constructors, only: &
+    conv1d, &
     conv2d, &
     dense, &
     dropout, &
@@ -11,8 +12,11 @@ module nf
     input, &
     layernorm, &
     linear2d, &
+    locally_connected1d, &
+    maxpool1d, &
     maxpool2d, &
     reshape, &
+    reshape2d, &
     self_attention
   use nf_loss, only: mse, quadratic
   use nf_metrics, only: corr, maxabs