diff options
Diffstat (limited to 'thirdparty/libtheora/x86/mmxloop.h')
| -rw-r--r-- | thirdparty/libtheora/x86/mmxloop.h | 271 |
1 files changed, 187 insertions, 84 deletions
diff --git a/thirdparty/libtheora/x86/mmxloop.h b/thirdparty/libtheora/x86/mmxloop.h index 2e870c795d..1f6090b567 100644 --- a/thirdparty/libtheora/x86/mmxloop.h +++ b/thirdparty/libtheora/x86/mmxloop.h @@ -9,88 +9,191 @@ On exit, mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)} and mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}; mm0 and mm3 are clobbered.*/ #define OC_LOOP_FILTER8_MMX \ - "#OC_LOOP_FILTER8_MMX\n\t" \ - /*mm7=0*/ \ - "pxor %%mm7,%%mm7\n\t" \ - /*mm6:mm0={a0,...,a7}*/ \ - "movq %%mm0,%%mm6\n\t" \ - "punpcklbw %%mm7,%%mm0\n\t" \ - "punpckhbw %%mm7,%%mm6\n\t" \ - /*mm3:mm5={d0,...,d7}*/ \ - "movq %%mm3,%%mm5\n\t" \ - "punpcklbw %%mm7,%%mm3\n\t" \ - "punpckhbw %%mm7,%%mm5\n\t" \ - /*mm6:mm0={a0-d0,...,a7-d7}*/ \ - "psubw %%mm3,%%mm0\n\t" \ - "psubw %%mm5,%%mm6\n\t" \ - /*mm3:mm1={b0,...,b7}*/ \ - "movq %%mm1,%%mm3\n\t" \ - "punpcklbw %%mm7,%%mm1\n\t" \ - "movq %%mm2,%%mm4\n\t" \ - "punpckhbw %%mm7,%%mm3\n\t" \ - /*mm5:mm4={c0,...,c7}*/ \ - "movq %%mm2,%%mm5\n\t" \ - "punpcklbw %%mm7,%%mm4\n\t" \ - "punpckhbw %%mm7,%%mm5\n\t" \ - /*mm7={3}x4 \ - mm5:mm4={c0-b0,...,c7-b7}*/ \ - "pcmpeqw %%mm7,%%mm7\n\t" \ - "psubw %%mm1,%%mm4\n\t" \ - "psrlw $14,%%mm7\n\t" \ - "psubw %%mm3,%%mm5\n\t" \ - /*Scale by 3.*/ \ - "pmullw %%mm7,%%mm4\n\t" \ - "pmullw %%mm7,%%mm5\n\t" \ - /*mm7={4}x4 \ - mm5:mm4=f={a0-d0+3*(c0-b0),...,a7-d7+3*(c7-b7)}*/ \ - "psrlw $1,%%mm7\n\t" \ - "paddw %%mm0,%%mm4\n\t" \ - "psllw $2,%%mm7\n\t" \ - "movq (%[ll]),%%mm0\n\t" \ - "paddw %%mm6,%%mm5\n\t" \ - /*R_i has the range [-127,128], so we compute -R_i instead. \ - mm4=-R_i=-(f+4>>3)=0xFF^(f-4>>3)*/ \ - "psubw %%mm7,%%mm4\n\t" \ - "psubw %%mm7,%%mm5\n\t" \ - "psraw $3,%%mm4\n\t" \ - "psraw $3,%%mm5\n\t" \ - "pcmpeqb %%mm7,%%mm7\n\t" \ - "packsswb %%mm5,%%mm4\n\t" \ - "pxor %%mm6,%%mm6\n\t" \ - "pxor %%mm7,%%mm4\n\t" \ - "packuswb %%mm3,%%mm1\n\t" \ - /*Now compute lflim of -mm4 cf. Section 7.10 of the sepc.*/ \ - /*There's no unsigned byte+signed byte with unsigned saturation op code, so \ - we have to split things by sign (the other option is to work in 16 bits, \ - but working in 8 bits gives much better parallelism). \ - We compute abs(R_i), but save a mask of which terms were negative in mm6. \ - Then we compute mm4=abs(lflim(R_i,L))=min(abs(R_i),max(2*L-abs(R_i),0)). \ - Finally, we split mm4 into positive and negative pieces using the mask in \ - mm6, and add and subtract them as appropriate.*/ \ - /*mm4=abs(-R_i)*/ \ - /*mm7=255-2*L*/ \ - "pcmpgtb %%mm4,%%mm6\n\t" \ - "psubb %%mm0,%%mm7\n\t" \ - "pxor %%mm6,%%mm4\n\t" \ - "psubb %%mm0,%%mm7\n\t" \ - "psubb %%mm6,%%mm4\n\t" \ - /*mm7=255-max(2*L-abs(R_i),0)*/ \ - "paddusb %%mm4,%%mm7\n\t" \ - /*mm4=min(abs(R_i),max(2*L-abs(R_i),0))*/ \ - "paddusb %%mm7,%%mm4\n\t" \ - "psubusb %%mm7,%%mm4\n\t" \ - /*Now split mm4 by the original sign of -R_i.*/ \ - "movq %%mm4,%%mm5\n\t" \ - "pand %%mm6,%%mm4\n\t" \ - "pandn %%mm5,%%mm6\n\t" \ - /*mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)}*/ \ - /*mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}*/ \ - "paddusb %%mm4,%%mm1\n\t" \ - "psubusb %%mm4,%%mm2\n\t" \ - "psubusb %%mm6,%%mm1\n\t" \ - "paddusb %%mm6,%%mm2\n\t" \ + "#OC_LOOP_FILTER8_MMX\n\t" \ + /*mm7=0*/ \ + "pxor %%mm7,%%mm7\n\t" \ + /*mm6:mm0={a0,...,a7}*/ \ + "movq %%mm0,%%mm6\n\t" \ + "punpcklbw %%mm7,%%mm0\n\t" \ + "punpckhbw %%mm7,%%mm6\n\t" \ + /*mm3:mm5={d0,...,d7}*/ \ + "movq %%mm3,%%mm5\n\t" \ + "punpcklbw %%mm7,%%mm3\n\t" \ + "punpckhbw %%mm7,%%mm5\n\t" \ + /*mm6:mm0={a0-d0,...,a7-d7}*/ \ + "psubw %%mm3,%%mm0\n\t" \ + "psubw %%mm5,%%mm6\n\t" \ + /*mm3:mm1={b0,...,b7}*/ \ + "movq %%mm1,%%mm3\n\t" \ + "punpcklbw %%mm7,%%mm1\n\t" \ + "movq %%mm2,%%mm4\n\t" \ + "punpckhbw %%mm7,%%mm3\n\t" \ + /*mm5:mm4={c0,...,c7}*/ \ + "movq %%mm2,%%mm5\n\t" \ + "punpcklbw %%mm7,%%mm4\n\t" \ + "punpckhbw %%mm7,%%mm5\n\t" \ + /*mm7={3}x4 \ + mm5:mm4={c0-b0,...,c7-b7}*/ \ + "pcmpeqw %%mm7,%%mm7\n\t" \ + "psubw %%mm1,%%mm4\n\t" \ + "psrlw $14,%%mm7\n\t" \ + "psubw %%mm3,%%mm5\n\t" \ + /*Scale by 3.*/ \ + "pmullw %%mm7,%%mm4\n\t" \ + "pmullw %%mm7,%%mm5\n\t" \ + /*mm7={4}x4 \ + mm5:mm4=f={a0-d0+3*(c0-b0),...,a7-d7+3*(c7-b7)}*/ \ + "psrlw $1,%%mm7\n\t" \ + "paddw %%mm0,%%mm4\n\t" \ + "psllw $2,%%mm7\n\t" \ + "movq (%[ll]),%%mm0\n\t" \ + "paddw %%mm6,%%mm5\n\t" \ + /*R_i has the range [-127,128], so we compute -R_i instead. \ + mm4=-R_i=-(f+4>>3)=0xFF^(f-4>>3)*/ \ + "psubw %%mm7,%%mm4\n\t" \ + "psubw %%mm7,%%mm5\n\t" \ + "psraw $3,%%mm4\n\t" \ + "psraw $3,%%mm5\n\t" \ + "pcmpeqb %%mm7,%%mm7\n\t" \ + "packsswb %%mm5,%%mm4\n\t" \ + "pxor %%mm6,%%mm6\n\t" \ + "pxor %%mm7,%%mm4\n\t" \ + "packuswb %%mm3,%%mm1\n\t" \ + /*Now compute lflim of -mm4 cf. Section 7.10 of the sepc.*/ \ + /*There's no unsigned byte+signed byte with unsigned saturation op code, so \ + we have to split things by sign (the other option is to work in 16 bits, \ + but working in 8 bits gives much better parallelism). \ + We compute abs(R_i), but save a mask of which terms were negative in mm6. \ + Then we compute mm4=abs(lflim(R_i,L))=min(abs(R_i),max(2*L-abs(R_i),0)). \ + Finally, we split mm4 into positive and negative pieces using the mask in \ + mm6, and add and subtract them as appropriate.*/ \ + /*mm4=abs(-R_i)*/ \ + /*mm7=255-2*L*/ \ + "pcmpgtb %%mm4,%%mm6\n\t" \ + "psubb %%mm0,%%mm7\n\t" \ + "pxor %%mm6,%%mm4\n\t" \ + "psubb %%mm0,%%mm7\n\t" \ + "psubb %%mm6,%%mm4\n\t" \ + /*mm7=255-max(2*L-abs(R_i),0)*/ \ + "paddusb %%mm4,%%mm7\n\t" \ + /*mm4=min(abs(R_i),max(2*L-abs(R_i),0))*/ \ + "paddusb %%mm7,%%mm4\n\t" \ + "psubusb %%mm7,%%mm4\n\t" \ + /*Now split mm4 by the original sign of -R_i.*/ \ + "movq %%mm4,%%mm5\n\t" \ + "pand %%mm6,%%mm4\n\t" \ + "pandn %%mm5,%%mm6\n\t" \ + /*mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)}*/ \ + /*mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}*/ \ + "paddusb %%mm4,%%mm1\n\t" \ + "psubusb %%mm4,%%mm2\n\t" \ + "psubusb %%mm6,%%mm1\n\t" \ + "paddusb %%mm6,%%mm2\n\t" \ -#define OC_LOOP_FILTER_V_MMX(_pix,_ystride,_ll) \ +/*On entry, mm0={a0,...,a7}, mm1={b0,...,b7}, mm2={c0,...,c7}, mm3={d0,...d7}. + On exit, mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)} and + mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}. + All other MMX registers are clobbered.*/ +#define OC_LOOP_FILTER8_MMXEXT \ + "#OC_LOOP_FILTER8_MMXEXT\n\t" \ + /*R_i=(a_i-3*b_i+3*c_i-d_i+4>>3) has the range [-127,128], so we compute \ + -R_i=(-a_i+3*b_i-3*c_i+d_i+3>>3) instead.*/ \ + /*This first part is based on the transformation \ + f = -(3*(c-b)+a-d+4>>3) \ + = -(3*(c+255-b)+(a+255-d)+4-1020>>3) \ + = -(3*(c+~b)+(a+~d)-1016>>3) \ + = 127-(3*(c+~b)+(a+~d)>>3) \ + = 128+~(3*(c+~b)+(a+~d)>>3) (mod 256). \ + Although pavgb(a,b) = (a+b+1>>1) (biased up), we rely heavily on the \ + fact that ~pavgb(~a,~b) = (a+b>>1) (biased down). \ + Using this, the last expression above can be computed in 8 bits of working \ + precision via: \ + u = ~pavgb(~b,c); \ + v = pavgb(b,~c); \ + This mask is 0 or 0xFF, and controls whether t is biased up or down: \ + m = u-v; \ + t = m^pavgb(m^~a,m^d); \ + f = 128+pavgb(pavgb(t,u),v); \ + This required some careful analysis to ensure that carries are propagated \ + correctly in all cases, but has been checked exhaustively.*/ \ + /*input (a, b, c, d, ., ., ., .)*/ \ + /*ff=0xFF; \ + u=b; \ + v=c; \ + ll=255-2*L;*/ \ + "pcmpeqb %%mm7,%%mm7\n\t" \ + "movq %%mm1,%%mm4\n\t" \ + "movq %%mm2,%%mm5\n\t" \ + "movq (%[ll]),%%mm6\n\t" \ + /*allocated u, v, ll, ff: (a, b, c, d, u, v, ll, ff)*/ \ + /*u^=ff; \ + v^=ff;*/ \ + "pxor %%mm7,%%mm4\n\t" \ + "pxor %%mm7,%%mm5\n\t" \ + /*allocated ll: (a, b, c, d, u, v, ll, ff)*/ \ + /*u=pavgb(u,c); \ + v=pavgb(v,b);*/ \ + "pavgb %%mm2,%%mm4\n\t" \ + "pavgb %%mm1,%%mm5\n\t" \ + /*u^=ff; \ + a^=ff;*/ \ + "pxor %%mm7,%%mm4\n\t" \ + "pxor %%mm7,%%mm0\n\t" \ + /*m=u-v;*/ \ + "psubb %%mm5,%%mm4\n\t" \ + /*freed u, allocated m: (a, b, c, d, m, v, ll, ff)*/ \ + /*a^=m; \ + d^=m;*/ \ + "pxor %%mm4,%%mm0\n\t" \ + "pxor %%mm4,%%mm3\n\t" \ + /*t=pavgb(a,d);*/ \ + "pavgb %%mm3,%%mm0\n\t" \ + "psllw $7,%%mm7\n\t" \ + /*freed a, d, ff, allocated t, of: (t, b, c, ., m, v, ll, of)*/ \ + /*t^=m; \ + u=m+v;*/ \ + "pxor %%mm4,%%mm0\n\t" \ + "paddb %%mm5,%%mm4\n\t" \ + /*freed t, m, allocated f, u: (f, b, c, ., u, v, ll, of)*/ \ + /*f=pavgb(f,u); \ + of=128;*/ \ + "pavgb %%mm4,%%mm0\n\t" \ + "packsswb %%mm7,%%mm7\n\t" \ + /*freed u, ff, allocated ll: (f, b, c, ., ll, v, ll, of)*/ \ + /*f=pavgb(f,v);*/ \ + "pavgb %%mm5,%%mm0\n\t" \ + "movq %%mm7,%%mm3\n\t" \ + "movq %%mm6,%%mm4\n\t" \ + /*freed v, allocated of: (f, b, c, of, ll, ., ll, of)*/ \ + /*Now compute lflim of R_i=-(128+mm0) cf. Section 7.10 of the sepc.*/ \ + /*There's no unsigned byte+signed byte with unsigned saturation op code, so \ + we have to split things by sign (the other option is to work in 16 bits, \ + but staying in 8 bits gives much better parallelism).*/ \ + /*Instead of adding the offset of 128 in mm3, we use it to split mm0. \ + This is the same number of instructions as computing a mask and splitting \ + after the lflim computation, but has shorter dependency chains.*/ \ + /*mm0=R_i<0?-R_i:0 (denoted abs(R_i<0))\ + mm3=R_i>0?R_i:0* (denoted abs(R_i>0))*/ \ + "psubusb %%mm0,%%mm3\n\t" \ + "psubusb %%mm7,%%mm0\n\t" \ + /*mm6=255-max(2*L-abs(R_i<0),0) \ + mm4=255-max(2*L-abs(R_i>0),0)*/ \ + "paddusb %%mm3,%%mm4\n\t" \ + "paddusb %%mm0,%%mm6\n\t" \ + /*mm0=min(abs(R_i<0),max(2*L-abs(R_i<0),0)) \ + mm3=min(abs(R_i>0),max(2*L-abs(R_i>0),0))*/ \ + "paddusb %%mm4,%%mm3\n\t" \ + "paddusb %%mm6,%%mm0\n\t" \ + "psubusb %%mm4,%%mm3\n\t" \ + "psubusb %%mm6,%%mm0\n\t" \ + /*mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)}*/ \ + /*mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}*/ \ + "paddusb %%mm3,%%mm1\n\t" \ + "psubusb %%mm3,%%mm2\n\t" \ + "psubusb %%mm0,%%mm1\n\t" \ + "paddusb %%mm0,%%mm2\n\t" \ + +#define OC_LOOP_FILTER_V(_filter,_pix,_ystride,_ll) \ do{ \ ptrdiff_t ystride3__; \ __asm__ __volatile__( \ @@ -104,7 +207,7 @@ "movq (%[pix],%[ystride]),%%mm1\n\t" \ /*mm2={c0,...,c7}*/ \ "movq (%[pix],%[ystride],2),%%mm2\n\t" \ - OC_LOOP_FILTER8_MMX \ + _filter \ /*Write it back out.*/ \ "movq %%mm1,(%[pix],%[ystride])\n\t" \ "movq %%mm2,(%[pix],%[ystride],2)\n\t" \ @@ -116,7 +219,7 @@ } \ while(0) -#define OC_LOOP_FILTER_H_MMX(_pix,_ystride,_ll) \ +#define OC_LOOP_FILTER_H(_filter,_pix,_ystride,_ll) \ do{ \ unsigned char *pix__; \ ptrdiff_t ystride3__; \ @@ -174,7 +277,7 @@ "punpckldq %%mm5,%%mm2\n\t" \ /*mm3=d7 d6 d5 d4 d3 d2 d1 d0*/ \ "punpckhdq %%mm5,%%mm3\n\t" \ - OC_LOOP_FILTER8_MMX \ + _filter \ /*mm2={b0+R_0'',...,b7+R_7''}*/ \ "movq %%mm1,%%mm0\n\t" \ /*mm1={b0+R_0'',c0-R_0'',...,b3+R_3'',c3-R_3''}*/ \ |