always extra

adeebshihadeh · adeebshihadeh · commit 995e341d7a41 · 2022-02-19T16:14:22.000-08:00
diff --git a/models/big_supercombo.dlc b/models/big_supercombo.dlc
diff --git a/models/big_supercombo.onnx b/models/big_supercombo.onnx
diff --git a/models/supercombo.dlc b/models/supercombo.dlc
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:209e9544e456dbc2a7d60490da65154e129bc84830909d8d931f97b3df93949b
-size 56684955
+oid sha256:ba3fe3e61853cc1434e3e220f40c8e9d1f1b9bab8458196ba3bea6a10b82c6ed
+size 72718099
diff --git a/models/supercombo.onnx b/models/supercombo.onnx
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2365bae967cce21ce68707c30bf2981bb7081ee5c3e6a3dff793e660f23ff622
-size 57554657
+oid sha256:bda57c1a66944f5a633ecd739a24d62702c717a234f2fdcc499dfa1d61c3c19e
+size 73147489
diff --git a/release/files_common b/release/files_common
@@ -58,7 +58,6 @@ common/transformations/transformations.pyx
 common/api/__init__.py
 
 models/supercombo.dlc
-models/big_supercombo.dlc
 models/dmonitoring_model_q.dlc
 
 release/*
diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript
@@ -31,9 +31,6 @@ thneed_src = [
 
 use_thneed = not GetOption('no_thneed')
 
-use_extra = True  # arch == "larch64"
-lenv['CXXFLAGS'].append('-DUSE_EXTRA=true' if use_extra else '-DUSE_EXTRA=false')
-
 if arch == "aarch64" or arch == "larch64":
   libs += ['gsl', 'CB']
   libs += ['gnustl_shared'] if arch == "aarch64" else ['pthread', 'dl']
@@ -68,7 +65,7 @@ common_model = lenv.Object(common_src)
 
 # build thneed model
 if use_thneed and arch in ("aarch64", "larch64"):
-  fn = "../../models/big_supercombo" if use_extra else "../../models/supercombo"
+  fn = "../../models/supercombo"
   compiler = lenv.Program('thneed/compile', ["thneed/compile.cc"]+common_model, LIBS=libs)
   cmd = f"cd {Dir('.').abspath} && {compiler[0].abspath} {fn}.dlc {fn}.thneed --binary"
 
diff --git a/selfdrive/modeld/modeld.cc b/selfdrive/modeld/modeld.cc
@@ -51,7 +51,7 @@ mat3 update_calibration(Eigen::Matrix<float, 3, 4> &extrinsics, bool wide_camera
   return matmul3(yuv_transform, transform);
 }
 
-void run_model(ModelState &model, VisionIpcClient &vipc_client_main, VisionIpcClient &vipc_client_extra, bool main_wide_camera, bool use_extra, bool use_extra_client) {
+void run_model(ModelState &model, VisionIpcClient &vipc_client_main, VisionIpcClient &vipc_client_extra, bool main_wide_camera, bool use_extra_client) {
   // messaging
   PubMaster pm({"modelV2", "cameraOdometry"});
   SubMaster sm({"lateralPlan", "roadCameraState", "liveCalibration"});
@@ -134,9 +134,7 @@ void run_model(ModelState &model, VisionIpcClient &vipc_client_main, VisionIpcCl
       }
 
       model_transform_main = update_calibration(extrinsic_matrix_eigen, main_wide_camera, false);
-      if (use_extra) {
-        model_transform_extra = update_calibration(extrinsic_matrix_eigen, Hardware::TICI(), true);
-      }
+      model_transform_extra = update_calibration(extrinsic_matrix_eigen, Hardware::TICI(), true);
       live_calib_seen = true;
     }
 
@@ -181,16 +179,15 @@ int main(int argc, char **argv) {
   }
 
   bool main_wide_camera = Hardware::TICI() ? Params().getBool("EnableWideCamera") : false;
-  bool use_extra = USE_EXTRA;
-  bool use_extra_client = Hardware::TICI() && use_extra && !main_wide_camera;
+  bool use_extra_client = Hardware::TICI() && !main_wide_camera;
 
   // cl init
   cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT);
   cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
 
   // init the models
   ModelState model;
-  model_init(&model, device_id, context, use_extra);
+  model_init(&model, device_id, context);
   LOGW("models loaded, modeld starting");
 
   VisionIpcClient vipc_client_main = VisionIpcClient("camerad", main_wide_camera ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD, true, device_id, context);
@@ -215,7 +212,7 @@ int main(int argc, char **argv) {
       LOGW("connected extra cam with buffer size: %d (%d x %d)", wb->len, wb->width, wb->height);
     }
 
-    run_model(model, vipc_client_main, vipc_client_extra, main_wide_camera, use_extra, use_extra_client);
+    run_model(model, vipc_client_main, vipc_client_extra, main_wide_camera, use_extra_client);
   }
 
   model_free(&model);
diff --git a/selfdrive/modeld/models/driving.cc b/selfdrive/modeld/models/driving.cc
@@ -26,19 +26,19 @@ constexpr const kj::ArrayPtr<const T> to_kj_array_ptr(const std::array<T, size>
   return kj::ArrayPtr(arr.data(), arr.size());
 }
 
-void model_init(ModelState* s, cl_device_id device_id, cl_context context, bool use_extra) {
+void model_init(ModelState* s, cl_device_id device_id, cl_context context) {
   s->frame = new ModelFrame(device_id, context);
   s->wide_frame = new ModelFrame(device_id, context);
 
 #ifdef USE_THNEED
-  s->m = std::make_unique<ThneedModel>(use_extra ? "../../models/big_supercombo.thneed" : "../../models/supercombo.thneed",
-   &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME, use_extra);
+  s->m = std::make_unique<ThneedModel>("../../models/supercombo.thneed",
+   &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME);
 #elif USE_ONNX_MODEL
-  s->m = std::make_unique<ONNXModel>(use_extra ? "../../models/big_supercombo.onnx" : "../../models/supercombo.onnx",
-   &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME, use_extra);
+  s->m = std::make_unique<ONNXModel>("../../models/supercombo.onnx",
+   &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME);
 #else
-  s->m = std::make_unique<SNPEModel>(use_extra ? "../../models/big_supercombo.dlc" : "../../models/supercombo.dlc",
-   &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME, use_extra);
+  s->m = std::make_unique<SNPEModel>("../../models/supercombo.dlc",
+   &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME);
 #endif
 
 #ifdef TEMPORAL
diff --git a/selfdrive/modeld/models/driving.h b/selfdrive/modeld/models/driving.h
@@ -266,7 +266,7 @@ struct ModelState {
 #endif
 };
 
-void model_init(ModelState* s, cl_device_id device_id, cl_context context, bool use_extra);
+void model_init(ModelState* s, cl_device_id device_id, cl_context context);
 ModelOutput *model_eval_frame(ModelState* s, VisionBuf* buf, VisionBuf* buf_wide,
                               const mat3 &transform, const mat3 &transform_wide, float *desire_in);
 void model_free(ModelState* s);
diff --git a/selfdrive/modeld/runners/onnxmodel.cc b/selfdrive/modeld/runners/onnxmodel.cc
@@ -14,12 +14,11 @@
 #include "selfdrive/common/swaglog.h"
 #include "selfdrive/common/util.h"
 
-ONNXModel::ONNXModel(const char *path, float *_output, size_t _output_size, int runtime, bool _use_extra) {
+ONNXModel::ONNXModel(const char *path, float *_output, size_t _output_size, int runtime) {
   LOGD("loading model %s", path);
 
   output = _output;
   output_size = _output_size;
-  use_extra = _use_extra;
 
   int err = pipe(pipein);
   assert(err == 0);
diff --git a/selfdrive/modeld/runners/onnxmodel.h b/selfdrive/modeld/runners/onnxmodel.h
@@ -6,7 +6,7 @@
 
 class ONNXModel : public RunModel {
 public:
-  ONNXModel(const char *path, float *output, size_t output_size, int runtime, bool use_extra = false);
+  ONNXModel(const char *path, float *output, size_t output_size, int runtime);
 	~ONNXModel();
   void addRecurrent(float *state, int state_size);
   void addDesire(float *state, int state_size);
@@ -30,7 +30,6 @@ class ONNXModel : public RunModel {
   int image_buf_size;
   float *extra_input_buf = NULL;
   int extra_buf_size;
-  bool use_extra;
 
   // pipe to communicate to keras subprocess
   void pread(float *buf, int size);
diff --git a/selfdrive/modeld/runners/snpemodel.cc b/selfdrive/modeld/runners/snpemodel.cc
@@ -14,10 +14,9 @@ void PrintErrorStringAndExit() {
   std::exit(EXIT_FAILURE);
 }
 
-SNPEModel::SNPEModel(const char *path, float *loutput, size_t loutput_size, int runtime, bool luse_extra) {
+SNPEModel::SNPEModel(const char *path, float *loutput, size_t loutput_size, int runtime) {
   output = loutput;
   output_size = loutput_size;
-  use_extra = luse_extra;
 #if defined(QCOM) || defined(QCOM2)
   if (runtime==USE_GPU_RUNTIME) {
     Runtime = zdl::DlSystem::Runtime_t::GPU;
@@ -91,7 +90,8 @@ SNPEModel::SNPEModel(const char *path, float *loutput, size_t loutput_size, int
     inputMap.add(input_tensor_name, inputBuffer.get());
   }
 
-  if (use_extra) {
+  // create extra input buffer
+  {
     const char *extra_tensor_name = strListi.at(1);
     const auto &extraDims_opt = snpe->getInputDimensions(extra_tensor_name);
     const zdl::DlSystem::TensorShape& bufferShape = *extraDims_opt;
@@ -153,7 +153,7 @@ void SNPEModel::addExtra(float *image_buf, int buf_size) {
 
 std::unique_ptr<zdl::DlSystem::IUserBuffer> SNPEModel::addExtra(float *state, int state_size, int idx) {
   // get input and output names
-  const auto real_idx = idx + (use_extra ? 1 : 0);
+  const auto real_idx = idx + 1;
   const auto &strListi_opt = snpe->getInputTensorNames();
   if (!strListi_opt) throw std::runtime_error("Error obtaining Input tensor names");
   const auto &strListi = *strListi_opt;
@@ -174,11 +174,9 @@ void SNPEModel::execute() {
     if (thneed == NULL) {
       bool ret = inputBuffer->setBufferAddress(input);
       assert(ret == true);
-      if (use_extra) {
-        assert(extra != NULL);
-        bool extra_ret = extraBuffer->setBufferAddress(extra);
-        assert(extra_ret == true);
-      }
+      assert(extra != NULL);
+      bool extra_ret = extraBuffer->setBufferAddress(extra);
+      assert(extra_ret == true);
       if (!snpe->execute(inputMap, outputMap)) {
         PrintErrorStringAndExit();
       }
@@ -216,22 +214,15 @@ void SNPEModel::execute() {
       }
       free(outputs_golden);
     } else {
-      if (use_extra) {
-        float *inputs[5] = {recurrent, trafficConvention, desire, extra, input};
-        thneed->execute(inputs, output);
-      } else {
-        float *inputs[4] = {recurrent, trafficConvention, desire, input};
-        thneed->execute(inputs, output);
-      }
+      float *inputs[5] = {recurrent, trafficConvention, desire, extra, input};
+      thneed->execute(inputs, output);
     }
   } else {
 #endif
     bool ret = inputBuffer->setBufferAddress(input);
     assert(ret == true);
-    if (use_extra) {
-      bool extra_ret = extraBuffer->setBufferAddress(extra);
-      assert(extra_ret == true);
-    }
+    bool extra_ret = extraBuffer->setBufferAddress(extra);
+    assert(extra_ret == true);
     if (!snpe->execute(inputMap, outputMap)) {
       PrintErrorStringAndExit();
     }
diff --git a/selfdrive/modeld/runners/snpemodel.h b/selfdrive/modeld/runners/snpemodel.h
@@ -22,7 +22,7 @@
 
 class SNPEModel : public RunModel {
 public:
-  SNPEModel(const char *path, float *loutput, size_t loutput_size, int runtime, bool luse_extra = false);
+  SNPEModel(const char *path, float *loutput, size_t loutput_size, int runtime);
   void addRecurrent(float *state, int state_size);
   void addTrafficConvention(float *state, int state_size);
   void addDesire(float *state, int state_size);
@@ -60,7 +60,6 @@ class SNPEModel : public RunModel {
   std::unique_ptr<zdl::DlSystem::IUserBuffer> extraBuffer;
   float *extra;
   size_t extra_size;
-  bool use_extra;
 
   // recurrent and desire
   std::unique_ptr<zdl::DlSystem::IUserBuffer> addExtra(float *state, int state_size, int idx);
diff --git a/selfdrive/modeld/runners/thneedmodel.cc b/selfdrive/modeld/runners/thneedmodel.cc
@@ -2,7 +2,7 @@
 
 #include <cassert>
 
-ThneedModel::ThneedModel(const char *path, float *loutput, size_t loutput_size, int runtime, bool luse_extra) {
+ThneedModel::ThneedModel(const char *path, float *loutput, size_t loutput_size, int runtime) {
   thneed = new Thneed(true);
   thneed->record = 0;
   thneed->load(path);
@@ -11,7 +11,6 @@ ThneedModel::ThneedModel(const char *path, float *loutput, size_t loutput_size,
 
   recorded = false;
   output = loutput;
-  use_extra = luse_extra;
 }
 
 void ThneedModel::addRecurrent(float *state, int state_size) {
@@ -35,8 +34,7 @@ void ThneedModel::addExtra(float *extra_input_buf, int buf_size) {
 }
 
 void* ThneedModel::getInputBuf() {
-  if (use_extra && thneed->input_clmem.size() > 4) return &(thneed->input_clmem[4]);
-  else if (!use_extra && thneed->input_clmem.size() > 3) return &(thneed->input_clmem[3]);
+  if (thneed->input_clmem.size() > 4) return &(thneed->input_clmem[4]);
   else return nullptr;
 }
 
@@ -48,26 +46,16 @@ void* ThneedModel::getExtraBuf() {
 void ThneedModel::execute() {
   if (!recorded) {
     thneed->record = THNEED_RECORD;
-    if (use_extra) {
-      float *inputs[5] = {recurrent, trafficConvention, desire, extra, input};
-      thneed->copy_inputs(inputs);
-    } else {
-      float *inputs[4] = {recurrent, trafficConvention, desire, input};
-      thneed->copy_inputs(inputs);
-    }
+    float *inputs[5] = {recurrent, trafficConvention, desire, extra, input};
+    thneed->copy_inputs(inputs);
     thneed->clexec();
     thneed->copy_output(output);
     thneed->stop();
 
     recorded = true;
   } else {
-    if (use_extra) {
-      float *inputs[5] = {recurrent, trafficConvention, desire, extra, input};
-      thneed->execute(inputs, output);
-    } else {
-      float *inputs[4] = {recurrent, trafficConvention, desire, input};
-      thneed->execute(inputs, output);
-    }
+    float *inputs[5] = {recurrent, trafficConvention, desire, extra, input};
+    thneed->execute(inputs, output);
   }
 }
 
diff --git a/selfdrive/modeld/runners/thneedmodel.h b/selfdrive/modeld/runners/thneedmodel.h
@@ -5,7 +5,7 @@
 
 class ThneedModel : public RunModel {
 public:
-  ThneedModel(const char *path, float *loutput, size_t loutput_size, int runtime, bool luse_extra = false);
+  ThneedModel(const char *path, float *loutput, size_t loutput_size, int runtime);
   void addRecurrent(float *state, int state_size);
   void addTrafficConvention(float *state, int state_size);
   void addDesire(float *state, int state_size);
@@ -17,7 +17,6 @@ class ThneedModel : public RunModel {
 private:
   Thneed *thneed = NULL;
   bool recorded;
-  bool use_extra;
 
   float *input;
   float *extra;
diff --git a/selfdrive/modeld/thneed/compile.cc b/selfdrive/modeld/thneed/compile.cc
@@ -13,7 +13,7 @@ int main(int argc, char* argv[]) {
   #define OUTPUT_SIZE 0x10000
 
   float *output = (float*)calloc(OUTPUT_SIZE, sizeof(float));
-  SNPEModel mdl(argv[1], output, 0, USE_GPU_RUNTIME, USE_EXTRA);
+  SNPEModel mdl(argv[1], output, 0, USE_GPU_RUNTIME);
 
   float state[TEMPORAL_SIZE] = {0};
   float desire[DESIRE_LEN] = {0};
@@ -25,9 +25,7 @@ int main(int argc, char* argv[]) {
   mdl.addDesire(desire, DESIRE_LEN);
   mdl.addTrafficConvention(traffic_convention, TRAFFIC_CONVENTION_LEN);
   mdl.addImage(input, 0);
-  if (USE_EXTRA) {
-    mdl.addExtra(extra, 0);
-  }
+  mdl.addExtra(extra, 0);
 
   // first run
   printf("************** execute 1 **************\n");
