Merge pull request #25 from emlearn/cnn-dequant

CNN: Fix wrong output values due to missing dequant

Author: jonnor

examples/mnist_cnn/README.md

@@ -55,7 +55,7 @@ mpremote mip install https://emlearn.github.io/emlearn-micropython/builds/master
 
 ```console
 mpremote cp mnist_cnn.tmdl :
-mpremote cp -r data/ :
+mpremote cp -r test_data/ :
 mpremote run mnist_cnn_run.py
 ```
 

examples/mnist_cnn/mnist_cnn.h

@@ -1,11 +1,13 @@
 
+import os
 import array
-import emlearn_cnn
 import time
 import gc
 
+import emlearn_cnn
+
 MODEL = 'mnist_cnn.tmdl'
-TEST_DATA_DIR = 'data/'
+TEST_DATA_DIR = 'test_data'
 
 def argmax(arr):
     idx_max = 0
@@ -30,6 +32,27 @@ def print_2d_buffer(arr, rowstride):
         gc.collect()
         print('\n')
 
+def load_images_from_directory(path):
+    sep = '/'
+
+    for filename in os.listdir(path):
+        # TODO: support standard image formats, like .bmp/.png/.jpeg
+        if not filename.endswith('.bin'):
+            continue
+
+        # Find the label (if any). The last part, X_label.format
+        label = None
+        basename = filename.split('.')[0]
+        tok = basename.split('_')
+        if len(tok) > 2:
+            label = tok[-1]
+
+        data_path = path + sep + filename
+        with open(data_path, 'rb') as f:
+            img = array.array('B', f.read())
+
+            yield img, label
+
 def test_cnn_mnist():
 
     # load model
@@ -42,22 +65,28 @@ def test_cnn_mnist():
     probabilities = array.array('f', (-1 for _ in range(out_length)))
 
     # run on some test data
-    for class_no in range(0, 10):
-        data_path = TEST_DATA_DIR + 'mnist_example_{0:d}.bin'.format(class_no)
-        #print('open', data_path)
-        with open(data_path, 'rb') as f:
-            img = array.array('B', f.read())
-
-            print_2d_buffer(img, 28)
-
-            run_start = time.ticks_us()
-            model.run(img, probabilities)
-            out = argmax(probabilities)
-            run_duration = time.ticks_diff(time.ticks_us(), run_start) / 1000.0 # ms
-
-            print('mnist-example-check', class_no, out, class_no == out, run_duration)
+    n_correct = 0
+    n_total = 0
+    for img, label in load_images_from_directory(TEST_DATA_DIR):
+        class_no = int(label) # mnist class labels are digits
+
+        #print_2d_buffer(img, 28)
+
+        run_start = time.ticks_us()
+        model.run(img, probabilities)
+        out = argmax(probabilities)
+        run_duration = time.ticks_diff(time.ticks_us(), run_start) / 1000.0 # ms
+        correct = class_no == out
+        n_total += 1
+        if correct:
+            n_correct += 1
+
+        print('mnist-example-check', class_no, '=', out, correct, round(run_duration, 3))
 
         gc.collect()
 
+    accuracy = n_correct / n_total
+    print('mnist-example-done', n_correct, '/', n_total, round(accuracy*100, ), '%')
+
 if __name__ == '__main__':
     test_cnn_mnist()

examples/mnist_cnn/mnist_cnn.h5

@@ -40,7 +40,9 @@ def train_mnist(h5_file, epochs=10):
     (x_orig_train, y_orig_train), (x_orig_test, y_orig_test) = mnist.load_data() 
     num_classes = 10
 
-    generate_test_files('test_data', x_orig_test, y_orig_test)
+    TEST_DATA_DIR = 'test_data'
+    generate_test_files(TEST_DATA_DIR, x_orig_test, y_orig_test)
+    print('Wrote test data to', TEST_DATA_DIR)
 
     x_train = x_orig_train
     x_test = x_orig_test
@@ -58,7 +60,7 @@ def train_mnist(h5_file, epochs=10):
 
     model.save(h5_file)
 
-def generate_test_files(out_dir, x, y):
+def generate_test_files(out_dir, x, y, samples_per_class=5):
 
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
@@ -71,15 +73,12 @@ def generate_test_files(out_dir, x, y):
     # select one per class
     for class_no in classes:
         matches = (Y_classes == class_no)
-        print('mm', matches.shape)
         x_matches = X_series[matches]
 
-        selected = x_matches.sample(n=1, random_state=1)
-        for s in selected:
-            print('ss', s.shape, s.dtype)
-            print(s)
-            out = os.path.join(out_dir, f'mnist_example_{class_no}.bin')
-            data = s.tobytes(order='C')
+        selected = x_matches.sample(n=samples_per_class, random_state=1)
+        for i, sample in enumerate(selected):
+            out = os.path.join(out_dir, f'mnist_example_{i}_{class_no}.bin')
+            data = sample.tobytes(order='C')
 
             assert len(data) == expect_bytes, (len(data), expect_bytes)
             with open(out, 'wb') as f:
@@ -94,9 +93,10 @@ def generate_tinymaix_model(h5_file,
         precision='fp32',
         quantize_data=None,
         quantize_type='0to1',
-        output_dequantize=False,
     ):
 
+    output_dequantize = quantize_data is not None
+
     # Convert .h5 to .tflite file
     assert h5_file.endswith('.h5'), 'Keras model HDF5 file must end with .h5'
     tflite_file = h5_file.replace('.h5', '.tflite')

examples/mnist_cnn/mnist_cnn.tflite

@@ -7,11 +7,14 @@
 #include <tinymaix.h>
 
 #include "tm_layers.c"
+//#include "tm_layers_O1.c"
 #include "tm_model.c"
 //#include "tm_stat.c"
 
 #include <string.h>
 
+#define DEBUG (1)
+
 
 // memset is used by some standard C constructs
 #if !defined(__linux__)
@@ -58,10 +61,13 @@ int TM_WEAK tm_get_outputs(tm_mdl_t* mdl, tm_mat_t* out, int out_length)
 
 static tm_err_t layer_cb(tm_mdl_t* mdl, tml_head_t* lh)
 {
+#if DEBUG
+    mp_printf(&mp_plat_print, "cnn-layer-cb type=%d \n", lh->type);
+#endif
+
     return TM_OK;
 }
 
-#define DEBUG (1)
 
 // MicroPython type
 typedef struct _mp_obj_mod_cnn_t {
@@ -209,12 +215,22 @@ static mp_obj_t mod_cnn_run(mp_obj_t self_obj, mp_obj_t input_obj, mp_obj_t outp
         mp_raise_ValueError(MP_ERROR_TEXT("run error"));
     }
 
-    // Copy output into
+    // Copy output
     tm_mat_t out = outs[0];
+
+#if DEBUG
+    mp_printf(&mp_plat_print, "cnn-run out.dims=(%d,%d,%d,%d) out.length=%d expect_length=%d \n",
+        out.dims, out.h, out.w, out.c, expect_out_length
+    );
+#endif
+
+    if (!((out.dims == 1) && (out.h == 1) && (out.w == 1) && out.c == expect_out_length)) {   
+        mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("unexpected output dims"));
+    }
+
     for(int i=0; i<expect_out_length; i++){
         output_buffer[i] = out.dataf[i];
     }
-
     return mp_const_none;
 }
 static MP_DEFINE_CONST_FUN_OBJ_3(mod_cnn_run_obj, mod_cnn_run);