Skip to content

Commit c982881

Browse files
DanLiu2IntelDanLiu2Intel
committed
add undefined type test
1 parent d0f5fc0 commit c982881

File tree

2 files changed

+343
-0
lines changed

2 files changed

+343
-0
lines changed

‎src/plugins/intel_npu/tests/functional/behavior/compile_and_infer_different_element_type.cpp‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (C) 2018-2025 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
5+
#include "behavior/compile_and_infer_different_element_type.hpp"
6+
#include "common/utils.hpp"
7+
#include "common/npu_test_env_cfg.hpp"
8+
#include "intel_npu/config/options.hpp"
9+
#include "intel_npu/npu_private_properties.hpp"
10+
11+
const std::vector<ov::AnyMap> configs = {};
12+
13+
INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests,
14+
InferRequestElementTypeTests,
15+
::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU),
16+
::testing::ValuesIn(configs)),
17+
ov::test::utils::appendPlatformTypeTestName<InferRequestElementTypeTests>);

‎src/plugins/intel_npu/tests/functional/behavior/compile_and_infer_different_element_type.hpp‎

Lines changed: 326 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,326 @@
1+
// Copyright (C) 2018-2025 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
5+
#include <string>
6+
#include <sstream>
7+
#include <vector>
8+
9+
#include "shared_test_classes/base/ov_behavior_test_utils.hpp"
10+
#include <common_test_utils/ov_tensor_utils.hpp>
11+
#include "openvino/pass/manager.hpp"
12+
#include "openvino/pass/serialize.hpp"
13+
#include "openvino/opsets/opset6.hpp"
14+
namespace ov {
15+
namespace test {
16+
namespace behavior {
17+
18+
using InferRequestElementTypeParams = std::tuple<
19+
std::string, // Device name
20+
ov::AnyMap // Config
21+
>;
22+
23+
class InferRequestElementTypeTests : public testing::WithParamInterface<InferRequestElementTypeParams>,
24+
public OVInferRequestTestBase {
25+
26+
public:
27+
static std::string getTestCaseName(testing::TestParamInfo<InferRequestElementTypeParams> obj) {
28+
std::string target_device;
29+
ov::AnyMap configuration;
30+
std::tie(target_device, configuration) = obj.param;
31+
std::replace(target_device.begin(), target_device.end(), ':', '.');
32+
std::ostringstream result;
33+
result << "targetDevice=" << target_device << "_";
34+
if (!configuration.empty()) {
35+
for (auto& configItem : configuration) {
36+
result << "configItem=" << configItem.first << "_";
37+
configItem.second.print(result);
38+
result << "_";
39+
}
40+
}
41+
return result.str();
42+
}
43+
44+
void SetUp() {
45+
// Skip test according to plugin specific disabledTestPatterns() (if any)
46+
SKIP_IF_CURRENT_TEST_IS_DISABLED()
47+
48+
std::tie(target_device, configuration) = this->GetParam();
49+
APIBaseTest::SetUp();
50+
}
51+
52+
protected:
53+
bool compareTensorOutputs(const ov::Tensor& dynamicInferenceOutput, const ov::Tensor& undefinedInferenceOutput);
54+
55+
std::shared_ptr<ov::Core> ie = utils::PluginCache::get().core();
56+
ov::AnyMap configuration;
57+
};
58+
59+
60+
bool InferRequestElementTypeTests::compareTensorOutputs(const ov::Tensor& dynamicInferenceOutput, const ov::Tensor& undefinedInferenceOutput) {
61+
const auto dynamicShape = dynamicInferenceOutput.get_shape();
62+
const auto undefinedShape = undefinedInferenceOutput.get_shape();
63+
64+
// compare two models' element types
65+
if (dynamicInferenceOutput.get_element_type() != undefinedInferenceOutput.get_element_type()) {
66+
return false;
67+
}
68+
69+
// compare two models' shapes
70+
if (dynamicShape.size() != undefinedShape.size()) {
71+
return false;
72+
}
73+
74+
if (!std::equal(dynamicShape.cbegin(), dynamicShape.cend(), undefinedShape.cbegin())) {
75+
return false;
76+
}
77+
// compare two models' data
78+
for (size_t i = 0; i < undefinedInferenceOutput.get_size(); i++) {
79+
if (fabs(dynamicInferenceOutput.data<float>()[i] - undefinedInferenceOutput.data<float>()[i]) >
80+
std::numeric_limits<float>::epsilon())
81+
return false;
82+
}
83+
return true;
84+
}
85+
86+
// Test whether the serialization and inference results of the dynamic type model and the undefined type model are the same
87+
TEST_P(InferRequestElementTypeTests, CompareDynamicAndUndefinedTypeNetwork) {
88+
// Skip test according to plugin specific disabledTestPatterns() (if any)
89+
SKIP_IF_CURRENT_TEST_IS_DISABLED()
90+
91+
// Customize a model with a dynamic type
92+
std::string dynamicTypeModelXmlString = R"V0G0N(<?xml version="1.0"?>
93+
<net name="custom_model" version="11">
94+
<layers>
95+
<layer id="0" name="Parameter_1" type="Parameter" version="opset1">
96+
<data shape="1,1,128" element_type="f32" />
97+
<output>
98+
<port id="0" precision="FP32" names="Parameter_1">
99+
<dim>1</dim>
100+
<dim>1</dim>
101+
<dim>128</dim>
102+
</port>
103+
</output>
104+
</layer>
105+
<layer id="1" name="Relu_2" type="ReLU" version="opset1">
106+
<input>
107+
<port id="0" precision="FP32">
108+
<dim>1</dim>
109+
<dim>1</dim>
110+
<dim>128</dim>
111+
</port>
112+
</input>
113+
<output>
114+
<port id="1" precision="FP32">
115+
<dim>1</dim>
116+
<dim>1</dim>
117+
<dim>128</dim>
118+
</port>
119+
</output>
120+
</layer>
121+
<layer id="2" name="ReadValue_3" type="ReadValue" version="opset6">
122+
<data variable_id="my_var" variable_type="dynamic" variable_shape="..." />
123+
<input>
124+
<port id="0" precision="FP32">
125+
<dim>1</dim>
126+
<dim>1</dim>
127+
<dim>128</dim>
128+
</port>
129+
</input>
130+
<output>
131+
<port id="1" precision="FP32">
132+
<dim>1</dim>
133+
<dim>1</dim>
134+
<dim>128</dim>
135+
</port>
136+
</output>
137+
</layer>
138+
<layer id="3" name="Assign_4" type="Assign" version="opset6">
139+
<data variable_id="my_var" />
140+
<input>
141+
<port id="0" precision="FP32">
142+
<dim>1</dim>
143+
<dim>1</dim>
144+
<dim>128</dim>
145+
</port>
146+
</input>
147+
<output>
148+
<port id="1" precision="FP32">
149+
<dim>1</dim>
150+
<dim>1</dim>
151+
<dim>128</dim>
152+
</port>
153+
</output>
154+
</layer>
155+
<layer id="4" name="Squeeze_5" type="Squeeze" version="opset1">
156+
<input>
157+
<port id="0" precision="FP32">
158+
<dim>1</dim>
159+
<dim>1</dim>
160+
<dim>128</dim>
161+
</port>
162+
</input>
163+
<output>
164+
<port id="1" precision="FP32" names="Output_5">
165+
<dim>128</dim>
166+
</port>
167+
</output>
168+
</layer>
169+
<layer id="5" name="Result_6" type="Result" version="opset1">
170+
<input>
171+
<port id="0" precision="FP32">
172+
<dim>128</dim>
173+
</port>
174+
</input>
175+
</layer>
176+
</layers>
177+
<edges>
178+
<edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
179+
<edge from-layer="1" from-port="1" to-layer="2" to-port="0" />
180+
<edge from-layer="2" from-port="1" to-layer="3" to-port="0" />
181+
<edge from-layer="3" from-port="1" to-layer="4" to-port="0" />
182+
<edge from-layer="4" from-port="1" to-layer="5" to-port="0" />
183+
</edges>
184+
<rt_info />
185+
</net>
186+
)V0G0N";
187+
188+
// Customize a model with a undefined type
189+
std::string undefinedTypeModelXmlString = R"V0G0N(<?xml version="1.0"?>
190+
<net name="custom_model" version="11">
191+
<layers>
192+
<layer id="0" name="Parameter_1" type="Parameter" version="opset1">
193+
<data shape="1,1,128" element_type="f32" />
194+
<output>
195+
<port id="0" precision="FP32" names="Parameter_1">
196+
<dim>1</dim>
197+
<dim>1</dim>
198+
<dim>128</dim>
199+
</port>
200+
</output>
201+
</layer>
202+
<layer id="1" name="Relu_2" type="ReLU" version="opset1">
203+
<input>
204+
<port id="0" precision="FP32">
205+
<dim>1</dim>
206+
<dim>1</dim>
207+
<dim>128</dim>
208+
</port>
209+
</input>
210+
<output>
211+
<port id="1" precision="FP32">
212+
<dim>1</dim>
213+
<dim>1</dim>
214+
<dim>128</dim>
215+
</port>
216+
</output>
217+
</layer>
218+
<layer id="2" name="ReadValue_3" type="ReadValue" version="opset6">
219+
<data variable_id="my_var" variable_type="undefined" variable_shape="..." />
220+
<input>
221+
<port id="0" precision="FP32">
222+
<dim>1</dim>
223+
<dim>1</dim>
224+
<dim>128</dim>
225+
</port>
226+
</input>
227+
<output>
228+
<port id="1" precision="FP32">
229+
<dim>1</dim>
230+
<dim>1</dim>
231+
<dim>128</dim>
232+
</port>
233+
</output>
234+
</layer>
235+
<layer id="3" name="Assign_4" type="Assign" version="opset6">
236+
<data variable_id="my_var" />
237+
<input>
238+
<port id="0" precision="FP32">
239+
<dim>1</dim>
240+
<dim>1</dim>
241+
<dim>128</dim>
242+
</port>
243+
</input>
244+
<output>
245+
<port id="1" precision="FP32">
246+
<dim>1</dim>
247+
<dim>1</dim>
248+
<dim>128</dim>
249+
</port>
250+
</output>
251+
</layer>
252+
<layer id="4" name="Squeeze_5" type="Squeeze" version="opset1">
253+
<input>
254+
<port id="0" precision="FP32">
255+
<dim>1</dim>
256+
<dim>1</dim>
257+
<dim>128</dim>
258+
</port>
259+
</input>
260+
<output>
261+
<port id="1" precision="FP32" names="Output_5">
262+
<dim>128</dim>
263+
</port>
264+
</output>
265+
</layer>
266+
<layer id="5" name="Result_6" type="Result" version="opset1">
267+
<input>
268+
<port id="0" precision="FP32">
269+
<dim>128</dim>
270+
</port>
271+
</input>
272+
</layer>
273+
</layers>
274+
<edges>
275+
<edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
276+
<edge from-layer="1" from-port="1" to-layer="2" to-port="0" />
277+
<edge from-layer="2" from-port="1" to-layer="3" to-port="0" />
278+
<edge from-layer="3" from-port="1" to-layer="4" to-port="0" />
279+
<edge from-layer="4" from-port="1" to-layer="5" to-port="0" />
280+
</edges>
281+
<rt_info />
282+
</net>
283+
)V0G0N";
284+
285+
std::stringstream dynamicTypeModelXmlStream, undefinedTypeModelXmlStream, dynamicTypeModelBinStream, undefinedTypeModelBinStream;
286+
287+
// Test whether the serialization results of the two models are the same
288+
auto dynamicTypeModel = ie->read_model(dynamicTypeModelXmlString, ov::Tensor());
289+
auto undefinedTypeModel = ie->read_model(undefinedTypeModelXmlString, ov::Tensor());
290+
291+
// compile the serialized models
292+
ov::pass::Serialize(dynamicTypeModelXmlStream, dynamicTypeModelBinStream).run_on_model(dynamicTypeModel);
293+
ov::pass::Serialize(undefinedTypeModelXmlStream, undefinedTypeModelBinStream).run_on_model(undefinedTypeModel);
294+
295+
ASSERT_TRUE(dynamicTypeModelXmlStream.str() == undefinedTypeModelXmlStream.str())
296+
<< "Serialized XML files are different: dynamic type vs undefined type";
297+
298+
// Test whether the inference results of the two models are the same
299+
// set input and output names
300+
const std::string inputName = "Parameter_1";
301+
const std::string outputName = "Output_5";
302+
303+
// create input tensor match the customized models
304+
ov::Shape shape = {1, 1, 128};
305+
ov::Tensor inTensor = ov::test::utils::create_and_fill_tensor(ov::element::f32, shape, 100, 0);
306+
307+
auto execNetDynamic = ie->compile_model(dynamicTypeModel, target_device, configuration);
308+
ov::InferRequest reqDynamic;
309+
OV_ASSERT_NO_THROW(reqDynamic = execNetDynamic.create_infer_request());
310+
OV_ASSERT_NO_THROW(reqDynamic.set_tensor(inputName, inTensor));
311+
OV_ASSERT_NO_THROW(reqDynamic.infer());
312+
313+
auto execNetUndefined = ie->compile_model(undefinedTypeModel, target_device, configuration);
314+
ov::InferRequest reqUndefined;
315+
OV_ASSERT_NO_THROW(reqUndefined = execNetUndefined.create_infer_request());
316+
OV_ASSERT_NO_THROW(reqUndefined.set_tensor(inputName, inTensor));
317+
OV_ASSERT_NO_THROW(reqUndefined.infer());
318+
319+
// compare the reference outputs between dynamic type model and undefined type model
320+
ASSERT_TRUE(compareTensorOutputs(reqDynamic.get_tensor(outputName), reqUndefined.get_tensor(outputName)))
321+
<< "Inference results are different: dynamic type vs undefined type";
322+
}
323+
324+
} // namespace behavior
325+
} // namespace test
326+
} // namespace ov

0 commit comments

Comments
 (0)