tensor cores in8 test failure #394
Description
This showed up in this CI job https://github.com/pytorch/ao/actions/runs/9557079384/job/26344629090?pr=282#step:12:2377 as we were trying to merge #282
CI was failing in an unrelated test because the tensor core flag was on as a global flag ae4ead1 - once it was removed the CI failure went way
test/integration/test_integration.py::TestSubclass::test_aq_int8_weight_only_quant_subclass_3_cuda Fatal Python error: Segmentation fault
Thread 0x00007f7fc63c0700 (most recent call first):
<no Python frame>
Thread 0x00007f7fc8dc1700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 47 in parse
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/jit.py", line 549 in parse
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1242 in ast_to_ttir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 112 in make_ir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 268 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fcd7c2700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1159 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1104 in visit_Attribute
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1042 in visit_Call
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 324 in visit_Return
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 306 in visit_compound_statement
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 391 in visit_FunctionDef
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 379 in generic_visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 314 in visit_Module
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1019 in call_JitFunction
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1054 in visit_Call
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 444 in visit_Assign
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 306 in visit_compound_statement
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 391 in visit_FunctionDef
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 379 in generic_visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 314 in visit_Module
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1242 in ast_to_ttir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 112 in make_ir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 268 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fd0bc4700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/cache.py", line 122 in put
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 275 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fd35c5700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/cache.py", line 122 in put
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 275 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fdd3c8700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/cache.py", line 123 in put
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 275 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fdddc9700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/language/semantic.py", line 654 in broadcast_impl_value
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/language/semantic.py", line 115 in binary_op_type_checking_impl
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/language/semantic.py", line 129 in add
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/language/core.py", line 756 in __add__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/language/core.py", line 34 in wrapper
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 485 in _apply_binary_method
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 498 in visit_BinOp
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 444 in visit_Assign
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 306 in visit_compound_statement
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 391 in visit_FunctionDef
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 379 in generic_visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 314 in visit_Module
File "/opt/conda/envs/venv/lib/python3.8/ast.py", line 371 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1149 in visit
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/code_generator.py", line 1242 in ast_to_ttir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 112 in make_ir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 268 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fe07ca700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/cache.py", line 123 in put
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 275 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7ece1fc700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/cache.py", line 122 in put
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 275 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fbfbbc700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/runtime/cache.py", line 122 in put
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 275 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Current thread 0x00007f7ed75fe700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/backends/nvidia/compiler.py", line 198 in make_llir
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/backends/nvidia/compiler.py", line 285 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/site-packages/triton/compiler/compiler.py", line 273 in compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 431 in _precompile_config
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 232 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/autotune_process.py", line 673 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 847 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1219 in precompile_with_captured_stdout
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1223 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 57 in run
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 80 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7e3cdfa700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fb29b7700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 78 in _worker
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f800a7da700 (most recent call first):
<no Python frame>
Thread 0x00007f8007dd9700 (most recent call first):
<no Python frame>
Thread 0x00007f800d21c700 (most recent call first):
<no Python frame>
Thread 0x00007f7fa97b2700 (most recent call first):
<no Python frame>
Thread 0x00007f7fa8db1700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 306 in wait
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 558 in wait
File "/opt/conda/envs/venv/lib/python3.8/site-packages/tqdm/_monitor.py", line 60 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f7fa83b0700 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 59 in _recv_msg
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 140 in _read_thread
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 870 in run
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 932 in _bootstrap_inner
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 890 in _bootstrap
Thread 0x00007f80c7ab0740 (most recent call first):
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 302 in wait
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 558 in wait
File "/opt/conda/envs/venv/lib/python3.8/threading.py", line 857 in start
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 213 in _adjust_thread_count
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/thread.py", line 188 in submit
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/_base.py", line 608 in <listcomp>
File "/opt/conda/envs/venv/lib/python3.8/concurrent/futures/_base.py", line 608 in map
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1222 in precompile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1307 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/select_algorithm.py", line 1653 in autotune_select_algorithm
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/kernel/mm.py", line 498 in tuned_mixed_mm
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/fx_passes/post_grad.py", line 325 in mixed_mm
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/graph.py", line 911 in call_function
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/fx/interpreter.py", line 203 in run_node
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/graph.py", line 1195 in run_node
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/fx/interpreter.py", line 146 in run
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/graph.py", line 715 in run
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 231 in time_wrapper
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_fx.py", line 824 in fx_codegen_and_compile
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_fx.py", line 535 in compile_fx_inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 231 in time_wrapper
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/debug.py", line 304 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/repro/after_aot.py", line 84 in debug_wrapper
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_fx.py", line 1421 in fw_compiler_base
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 231 in time_wrapper
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_functorch/_aot_autograd/jit_compile_runtime_wrappers.py", line 488 in aot_dispatch_autograd
Traceback (most recent call last):
File "/home/ec2-user/actions-runner/_work/ao/ao/test-infra/.github/scripts/run_with_env_secrets.py", line 101, in <module>
main()
File "/home/ec2-user/actions-runner/_work/ao/ao/test-infra/.github/scripts/run_with_env_secrets.py", line 97, in main
run_cmd_or_die(f"docker exec -t {container_name} /exec")
File "/home/ec2-user/actions-runner/_work/ao/ao/test-infra/.github/scripts/run_with_env_secrets.py", line 38, in run_cmd_or_die
raise RuntimeError(f"Command {cmd} failed with exit code {exit_code}")
RuntimeError: Command docker exec -t 5286d0a91c6239f67896fdd6290b951ef405f4029bc139c9580078cc2d272201 /exec failed with exit code 139
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_functorch/aot_autograd.py", line 696 in create_aot_dispatcher_function
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 231 in time_wrapper
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_functorch/aot_autograd.py", line 964 in dispatch_and_compile
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_functorch/aot_autograd.py", line 975 in aot_module_simplified
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/backends/common.py", line 69 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_fx.py", line 1516 in compile_fx
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_inductor/compile_fx.py", line 1272 in compile_fx
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/__init__.py", line 2172 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/repro/after_dynamo.py", line 129 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1394 in call_user_compiler
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 231 in time_wrapper
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1322 in compile_and_call_fx_graph
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/output_graph.py", line 1100 in compile_subgraph
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2638 in _return
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 2653 in RETURN_VALUE
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 816 in step
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line 904 in run
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/symbolic_convert.py", line [2462](https://github.com/pytorch/ao/actions/runs/9557079384/job/26344629090?pr=282#step:12:2463) in run
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 582 in transform
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 178 in _fn
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/bytecode_transformation.py", line 1185 in transform_code_object
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 636 in compile_inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/utils.py", line 231 in time_wrapper
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 817 in _compile
File "/opt/conda/envs/venv/lib/python3.8/contextlib.py", line 75 in inner
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_strobelight/compile_time_profiler.py", line 129 in profile_compile_time
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_utils_internal.py", line 84 in wrapper_function
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 472 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 948 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/convert_frame.py", line 1116 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/_dynamo/eval_frame.py", line 433 in _fn
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1575 in _call_impl
File "/opt/conda/envs/venv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1566 in _wrapped_call_impl
File "/pytorch/ao/test/integration/test_integration.py", line 676 in _test_lin_weight_subclass_impl
File "/pytorch/ao/test/integration/test_integration.py", line 147 in wrapper
File "/pytorch/ao/test/integration/test_integration.py", line 710 in test_aq_int8_weight_only_quant_subclass
File "/opt/conda/envs/venv/lib/python3.8/site-packages/parameterized/parameterized.py", line 620 in standalone_func
File "/opt/conda/envs/venv/lib/python3.8/unittest/case.py", line 633 in _callTestMethod
File "/opt/conda/envs/venv/lib/python3.8/unittest/case.py", line 676 in run
File "/opt/conda/envs/venv/lib/python3.8/unittest/case.py", line 736 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/unittest.py", line 333 in runtest
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 169 in pytest_runtest_call
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 262 in <lambda>
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 341 in from_call
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 261 in call_runtest_hook
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 222 in call_and_report
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 133 in runtestprotocol
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/runner.py", line 114 in pytest_runtest_protocol
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/main.py", line 349 in pytest_runtestloop
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/main.py", line 324 in _main
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/main.py", line 270 in wrap_session
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/main.py", line 317 in pytest_cmdline_main
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/opt/conda/envs/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/config/__init__.py", line 166 in main
File "/opt/conda/envs/venv/lib/python3.8/site-packages/_pytest/config/__init__.py", line 189 in console_main
File "/opt/conda/envs/venv/bin/pytest", line 8 in <module>
/exec: line 14: 452 Segmentation fault pytest test --verbose -s
Error: Process completed with exit code 1.