Opened 20 months ago
Closed 18 months ago
#10234 closed defect (fixed)
Assertion qmin <= qmax at ratecontrol.c:123
| Reported by: | Youngseok Choi | Owned by: | |
|---|---|---|---|
| Priority: | important | Component: | avcodec |
| Version: | git-master | Keywords: | |
| Cc: | Blocked By: | ||
| Blocking: | Reproduced by developer: | yes | |
| Analyzed by developer: | no |
Description
Hi, we are developing a new fuzz testing feature, and it found a assertion violation on ffmpeg.
How to reproduce:
% ./ffmpeg -i <input_file> -f mp4 -lmax 1 e ffmpeg version N-109968-gcc76e8340d (git-master) built on Ubuntu 18.04.1 with gcc 7.5.0
You can download <input_file> from https://github.com/3-24/oss-fuzz-reports/raw/master/ffmpeg/poc_1/poc_file.
Command output:
ffmpeg version N-109968-gcc76e8340d Copyright (c) 2000-2023 the FFmpeg developers
built with gcc 7 (Ubuntu 7.5.0-3ubuntu1~18.04)
configuration: --extra-cflags='-fsanitize=address -g -O0' --extra-cxxflags='-fsanitize=address -g -O0' --extra-ldflags='-fsanitize=address -g -O0'
libavutil 58. 3.100 / 58. 3.100
libavcodec 60. 6.100 / 60. 6.100
libavformat 60. 4.100 / 60. 4.100
libavdevice 60. 2.100 / 60. 2.100
libavfilter 9. 4.100 / 9. 4.100
libswscale 7. 2.100 / 7. 2.100
libswresample 4. 11.100 / 4. 11.100
[h261 @ 0x617000000080] Format h261 detected only with low score of 25, misdetection possible!
[h261 @ 0x619000000580] warning: first frame is no keyframe
[h261 @ 0x619000000580] illegal ac vlc code at 6x0
[h261 @ 0x619000000580] Error at MB: 6
Input #0, h261, from 'poc_file':
Duration: N/A, bitrate: N/A
Stream #0:0: Video: h261, yuv420p, 176x144, 29.97 tbr, 1200k tbn
Stream mapping:
Stream #0:0 -> #0:0 (h261 (native) -> mpeg4 (native))
Press [q] to stop, [?] for help
[h261 @ 0x619000001980] warning: first frame is no keyframe
[h261 @ 0x619000001980] illegal ac vlc code at 6x0
[h261 @ 0x619000001980] Error at MB: 6
[mpeg4 @ 0x619000002d80] too many threads/slices (10), reducing to 9
Output #0, mp4, to 'e':
Metadata:
encoder : Lavf60.4.100
Stream #0:0: Video: mpeg4 (mp4v / 0x7634706D), yuv420p(progressive), 176x144, q=2-31, 200 kb/s, 29.97 fps, 30k tbn
Metadata:
encoder : Lavc60.6.100 mpeg4
Side data:
cpb: bitrate max/min/avg: 0/0/200000 buffer size: 0 vbv_delay: N/A
Assertion qmin <= qmax failed at libavcodec/ratecontrol.c:123
Aborted (core dumped)
Backtrace:
#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007ffff56207f1 in __GI_abort () at abort.c:79
#2 0x00005555571fef60 in get_qminmax (s=0x625000005100, s=0x625000005100, s=0x625000005100, pict_type=1, qmax_ret=<synthetic pointer>,
qmin_ret=<synthetic pointer>) at libavcodec/ratecontrol.c:123
#3 ff_rate_estimate_qscale (s=s@entry=0x625000005100, dry_run=<optimized out>) at libavcodec/ratecontrol.c:885
#4 0x0000555556fd5cb7 in estimate_qp (s=s@entry=0x625000005100, dry_run=dry_run@entry=0) at libavcodec/mpegvideo_enc.c:3525
#5 0x0000555556fd9666 in encode_picture (s=0x625000005100) at libavcodec/mpegvideo_enc.c:3721
#6 ff_mpv_encode_picture (avctx=<optimized out>, pkt=<optimized out>, pic_arg=<optimized out>, got_packet=<optimized out>)
at libavcodec/mpegvideo_enc.c:1801
#7 0x0000555556a8659b in ff_encode_encode_cb (avctx=avctx@entry=0x619000002d80, avpkt=avpkt@entry=0x610000002640, frame=0x616000011d80,
got_packet=got_packet@entry=0x7fffffffcb80) at libavcodec/encode.c:223
#8 0x0000555556a872e6 in encode_simple_internal (avpkt=0x610000002640, avctx=0x619000002d80) at libavcodec/encode.c:309
#9 encode_simple_receive_packet (avpkt=<optimized out>, avctx=<optimized out>) at libavcodec/encode.c:323
#10 encode_receive_packet_internal (avctx=avctx@entry=0x619000002d80, avpkt=0x610000002640) at libavcodec/encode.c:357
#11 0x0000555556a87913 in avcodec_send_frame (avctx=0x619000002d80, frame=0x616000009080) at libavcodec/encode.c:506
#12 0x0000555555c6dd2d in encode_frame (of=<optimized out>, ost=0x618000000080, frame=<optimized out>) at fftools/ffmpeg.c:904
#13 0x0000555555c719fe in submit_encode_frame (frame=0x616000009080, ost=0x618000000080, of=0x611000000900) at fftools/ffmpeg.c:985
#14 do_video_out (of=0x611000000900, ost=0x618000000080, next_picture=<optimized out>) at fftools/ffmpeg.c:1340
#15 0x0000555555c7335e in reap_filters (flush=<optimized out>) at fftools/ffmpeg.c:1426
#16 0x0000555555c7b01b in transcode_step () at fftools/ffmpeg.c:4002
#17 transcode () at fftools/ffmpeg.c:4039
#18 0x0000555555bed03e in main (argc=8, argv=0x7fffffffe0d8) at fftools/ffmpeg.c:4177
Disassembly around pc:
(gdb) disass $pc-32,$pc+32 Dump of assembler code from 0x7ffff561ee67 to 0x7ffff561eea7: 0x00007ffff561ee67 <__GI_raise+167>: add %dh,%al 0x00007ffff561ee69 <__GI_raise+169>: (bad) 0x00007ffff561ee6a <__GI_raise+170>: pushq 0x3b(%rdi) 0x00007ffff561ee6d <__GI_raise+173>: mov %eax,%r8d 0x00007ffff561ee70 <__GI_raise+176>: mov $0x8,%r10d 0x00007ffff561ee76 <__GI_raise+182>: xor %edx,%edx 0x00007ffff561ee78 <__GI_raise+184>: mov %r9,%rsi 0x00007ffff561ee7b <__GI_raise+187>: mov $0x2,%edi 0x00007ffff561ee80 <__GI_raise+192>: mov $0xe,%eax 0x00007ffff561ee85 <__GI_raise+197>: syscall => 0x00007ffff561ee87 <__GI_raise+199>: mov 0x108(%rsp),%rcx 0x00007ffff561ee8f <__GI_raise+207>: xor %fs:0x28,%rcx 0x00007ffff561ee98 <__GI_raise+216>: mov %r8d,%eax 0x00007ffff561ee9b <__GI_raise+219>: jne 0x7ffff561eebc <__GI_raise+252> 0x00007ffff561ee9d <__GI_raise+221>: add $0x118,%rsp 0x00007ffff561eea4 <__GI_raise+228>: retq 0x00007ffff561eea5 <__GI_raise+229>: nopl (%rax)
register info:
rax 0x0 0
rbx 0xec 236
rcx 0x7ffff561ee87 140737310224007
rdx 0x0 0
rsi 0x7fffffffc2a0 140737488339616
rdi 0x2 2
rbp 0x619000002d80 0x619000002d80
rsp 0x7fffffffc2a0 0x7fffffffc2a0
r8 0x0 0
r9 0x7fffffffc2a0 140737488339616
r10 0x8 8
r11 0x246 582
r12 0x625000005100 108095736926464
r13 0x619000002d80 107271103196544
r14 0x1 1
r15 0x7fffffffc5e0 140737488340448
rip 0x7ffff561ee87 0x7ffff561ee87 <__GI_raise+199>
eflags 0x246 [ PF ZF IF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
st0 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st1 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st2 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st3 -nan(0x8080808080808080) (raw 0xffff8080808080808080)
st4 <invalid float value> (raw 0xffff0000000000000000)
st5 <invalid float value> (raw 0xffff0005000500050005)
st6 <invalid float value> (raw 0xffff000a000a000a000a)
st7 -nan(0xfffbfffbfffbfffb) (raw 0xfffffffbfffbfffbfffb)
fctrl 0x37f 895
fstat 0x0 0
ftag 0xffff 65535
fiseg 0x0 0
fioff 0x0 0
foseg 0x0 0
fooff 0x0 0
fop 0x0 0
mxcsr 0x1fa8 [ OE PE IM DM ZM OM UM PM ]
ymm0 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 = {
0xff <repeats 16 times>, 0x0 <repeats 16 times>}, v16_int16 = {0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0}, v8_int32 = {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xffffffffffffffff,
0xffffffffffffffff, 0x0, 0x0}, v2_int128 = {0xffffffffffffffffffffffffffffffff, 0x0}}
ymm1 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x25 <repeats 16 times>,
0x0 <repeats 16 times>}, v16_int16 = {0x2525, 0x2525, 0x2525, 0x2525, 0x2525, 0x2525, 0x2525, 0x2525, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v8_int32 = {0x25252525, 0x25252525, 0x25252525, 0x25252525, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x2525252525252525, 0x2525252525252525, 0x0, 0x0},
v2_int128 = {0x25252525252525252525252525252525, 0x0}}
ymm2 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 = {0x0,
0x0, 0xff, 0xff, 0x0, 0x0, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0xff, 0x0 <repeats 16 times>}, v16_int16 = {0x0, 0xffff, 0x0, 0xffff,
0x0, 0x0, 0x0, 0xffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0xffff0000, 0xffff0000, 0x0, 0xffff0000, 0x0, 0x0, 0x0, 0x0},
v4_int64 = {0xffff0000ffff0000, 0xffff000000000000, 0x0, 0x0}, v2_int128 = {0xffff000000000000ffff0000ffff0000, 0x0}}
ymm3 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
---Type <return> to continue, or q <return> to quit---
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0, 0x0}}
ymm4 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0}, v32_int8 = {0x0,
0x0, 0x0, 0x0, 0xff <repeats 12 times>, 0x0 <repeats 16 times>}, v16_int16 = {0x0, 0x0, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x0, 0xffffffff, 0xffffffff, 0xffffffff, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0xffffffff00000000,
0xffffffffffffffff, 0x0, 0x0}, v2_int128 = {0xffffffffffffffffffffffff00000000, 0x0}}
ymm5 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0, 0x0}}
ymm6 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0, 0x0}}
ymm7 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0, 0x0}}
ymm8 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {0x0, 0x0}}
ymm9 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x80, 0x0, 0x80, 0x0, 0x80, 0x0,
0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0 <repeats 17 times>}, v16_int16 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x800080, 0x800080, 0x800080, 0x800080, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x80008000800080,
0x80008000800080, 0x0, 0x0}, v2_int128 = {0x800080008000800080008000800080, 0x0}}
ymm10 {v8_float = {0x0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x7fffffffffffffff, 0x0, 0x0, 0x0}, v32_int8 = {0x84, 0x78,
0x84, 0x83, 0x7d, 0x7d, 0x89, 0x7b, 0x87, 0x76, 0x7f, 0x86, 0x83, 0x78, 0x81, 0x81, 0x0 <repeats 16 times>}, v16_int16 = {0x7884, 0x8384, 0x7d7d,
0x7b89, 0x7687, 0x867f, 0x7883, 0x8181, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x83847884, 0x7b897d7d, 0x867f7687, 0x81817883, 0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x7b897d7d83847884, 0x81817883867f7687, 0x0, 0x0}, v2_int128 = {0x81817883867f76877b897d7d83847884, 0x0}}
ymm11 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x87, 0x0, 0x76, 0x0, 0x7f, 0x0,
0x86, 0x0, 0x83, 0x0, 0x78, 0x0, 0x81, 0x0, 0x81, 0x0 <repeats 17 times>}, v16_int16 = {0x87, 0x76, 0x7f, 0x86, 0x83, 0x78, 0x81, 0x81, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x760087, 0x86007f, 0x780083, 0x810081, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x86007f00760087,
0x81008100780083, 0x0, 0x0}, v2_int128 = {0x810081007800830086007f00760087, 0x0}}
ymm12 {v8_float = {0xffffffff, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x7fffffffffffffff, 0x0, 0x0, 0x0}, v32_int8 = {0x7d,
0x83, 0x84, 0x7a, 0x82, 0x81, 0x82, 0x7e, 0x85, 0x7a, 0x7d, 0x82, 0x88, 0x79, 0x7c, 0x85, 0x0 <repeats 16 times>}, v16_int16 = {0x837d, 0x7a84,
0x8182, 0x7e82, 0x7a85, 0x827d, 0x7988, 0x857c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x7a84837d, 0x7e828182, 0x827d7a85,
0x857c7988, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x7e8281827a84837d, 0x857c7988827d7a85, 0x0, 0x0}, v2_int128 = {0x857c7988827d7a857e8281827a84837d,
0x0}}
ymm13 {v8_float = {0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x7fffffffffffffff, 0x7fffffffffffffff,
0x0, 0x0}, v32_int8 = {0x78, 0x89, 0x88, 0x74, 0x80, 0x85, 0x84, 0x7b, 0x76, 0x8b, 0x8a, 0x72, 0x7d, 0x87, 0x87, 0x79, 0x0 <repeats 16 times>},
v16_int16 = {0x8978, 0x7488, 0x8580, 0x7b84, 0x8b76, 0x728a, 0x877d, 0x7987, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x74888978,
0x7b848580, 0x728a8b76, 0x7987877d, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x7b84858074888978, 0x7987877d728a8b76, 0x0, 0x0}, v2_int128 = {
0x7987877d728a8b767b84858074888978, 0x0}}
ymm14 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x80, 0x0, 0x80, 0x0, 0x80, 0x0,
0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0 <repeats 17 times>}, v16_int16 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x800080, 0x800080, 0x800080, 0x800080, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x80008000800080,
0x80008000800080, 0x0, 0x0}, v2_int128 = {0x800080008000800080008000800080, 0x0}}
ymm15 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x80, 0x0, 0x80, 0x0, 0x80, 0x0,
0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0, 0x80, 0x0 <repeats 17 times>}, v16_int16 = {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x800080, 0x800080, 0x800080, 0x800080, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x80008000800080,
0x80008000800080, 0x0, 0x0}, v2_int128 = {0x800080008000800080008000800080, 0x0}}
Change History (1)
comment:1 by , 18 months ago
| Component: | ffmpeg → avcodec |
|---|---|
| Priority: | normal → important |
| Reproduced by developer: | set |
| Resolution: | → fixed |
| Status: | new → closed |
Note:
See TracTickets
for help on using tickets.
fixed in 13450b67229540fd79075a84185b29db1eec2687.