godot/drivers/webp/dsp/enc_mips_dsp_r2.c

// Copyright 2014 Google Inc. All Rights Reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the COPYING file in the root of the source
// tree. An additional intellectual property rights grant can be found
// in the file PATENTS. All contributing project authors may
// be found in the AUTHORS file in the root of the source tree.
// -----------------------------------------------------------------------------
//
// MIPS version of speed-critical encoding functions.
//
// Author(s): Darko Laus ([email protected])
//            Mirko Raus ([email protected])

#include "./dsp.h"

#if defined(WEBP_USE_MIPS_DSP_R2)

#include "./mips_macro.h"
#include "../enc/cost.h"
#include "../enc/vp8enci.h"

static const int kC1 = 20091 + (1 << 16);
static const int kC2 = 35468;

// O - output
// I - input (macro doesn't change it)
#define ADD_SUB_HALVES_X4(O0, O1, O2, O3, O4, O5, O6, O7,                      \
                          I0, I1, I2, I3, I4, I5, I6, I7)                      \
  "addq.ph          %[" #O0 "],   %[" #I0 "],  %[" #I1 "]     \n\t"            \
  "subq.ph          %[" #O1 "],   %[" #I0 "],  %[" #I1 "]     \n\t"            \
  "addq.ph          %[" #O2 "],   %[" #I2 "],  %[" #I3 "]     \n\t"            \
  "subq.ph          %[" #O3 "],   %[" #I2 "],  %[" #I3 "]     \n\t"            \
  "addq.ph          %[" #O4 "],   %[" #I4 "],  %[" #I5 "]     \n\t"            \
  "subq.ph          %[" #O5 "],   %[" #I4 "],  %[" #I5 "]     \n\t"            \
  "addq.ph          %[" #O6 "],   %[" #I6 "],  %[" #I7 "]     \n\t"            \
  "subq.ph          %[" #O7 "],   %[" #I6 "],  %[" #I7 "]     \n\t"

// IO - input/output
#define ABS_X8(IO0, IO1, IO2, IO3, IO4, IO5, IO6, IO7)                         \
  "absq_s.ph        %[" #IO0 "],   %[" #IO0 "]                \n\t"            \
  "absq_s.ph        %[" #IO1 "],   %[" #IO1 "]                \n\t"            \
  "absq_s.ph        %[" #IO2 "],   %[" #IO2 "]                \n\t"            \
  "absq_s.ph        %[" #IO3 "],   %[" #IO3 "]                \n\t"            \
  "absq_s.ph        %[" #IO4 "],   %[" #IO4 "]                \n\t"            \
  "absq_s.ph        %[" #IO5 "],   %[" #IO5 "]                \n\t"            \
  "absq_s.ph        %[" #IO6 "],   %[" #IO6 "]                \n\t"            \
  "absq_s.ph        %[" #IO7 "],   %[" #IO7 "]                \n\t"

// dpa.w.ph $ac0 temp0 ,temp1
//  $ac += temp0[31..16] * temp1[31..16] + temp0[15..0] * temp1[15..0]
// dpax.w.ph $ac0 temp0 ,temp1
//  $ac += temp0[31..16] * temp1[15..0] + temp0[15..0] * temp1[31..16]
// O - output
// I - input (macro doesn't change it)
#define MUL_HALF(O0, I0, I1, I2, I3, I4, I5, I6, I7,                           \
                 I8, I9, I10, I11, I12, I13, I14, I15)                         \
    "mult            $ac0,      $zero,     $zero              \n\t"            \
    "dpa.w.ph        $ac0,      %[" #I2 "],  %[" #I0 "]       \n\t"            \
    "dpax.w.ph       $ac0,      %[" #I5 "],  %[" #I6 "]       \n\t"            \
    "dpa.w.ph        $ac0,      %[" #I8 "],  %[" #I9 "]       \n\t"            \
    "dpax.w.ph       $ac0,      %[" #I11 "], %[" #I4 "]       \n\t"            \
    "dpa.w.ph        $ac0,      %[" #I12 "], %[" #I7 "]       \n\t"            \
    "dpax.w.ph       $ac0,      %[" #I13 "], %[" #I1 "]       \n\t"            \
    "dpa.w.ph        $ac0,      %[" #I14 "], %[" #I3 "]       \n\t"            \
    "dpax.w.ph       $ac0,      %[" #I15 "], %[" #I10 "]      \n\t"            \
    "mflo            %[" #O0 "],  $ac0                        \n\t"

#define OUTPUT_EARLY_CLOBBER_REGS_17()                                         \
  OUTPUT_EARLY_CLOBBER_REGS_10(),                                              \
  [temp11]"=&r"(temp11), [temp12]"=&r"(temp12), [temp13]"=&r"(temp13),         \
  [temp14]"=&r"(temp14), [temp15]"=&r"(temp15), [temp16]"=&r"(temp16),         \
  [temp17]"=&r"(temp17)

// macro for one horizontal pass in FTransform
// temp0..temp15 holds tmp[0]..tmp[15]
// A - offset in bytes to load from src and ref buffers
// TEMP0..TEMP3 - registers for corresponding tmp elements
#define HORIZONTAL_PASS(A, TEMP0, TEMP1, TEMP2, TEMP3)                         \
  "lw              %[" #TEMP0 "],   0(%[args])                          \n\t"  \
  "lw              %[" #TEMP1 "],   4(%[args])                          \n\t"  \
  "lw              %[" #TEMP2 "],   " XSTR(BPS) "*" #A "(%[" #TEMP0 "]) \n\t"  \
  "lw              %[" #TEMP3 "],   " XSTR(BPS) "*" #A "(%[" #TEMP1 "]) \n\t"  \
  "preceu.ph.qbl   %[" #TEMP0 "],   %[" #TEMP2 "]                       \n\t"  \
  "preceu.ph.qbl   %[" #TEMP1 "],   %[" #TEMP3 "]                       \n\t"  \
  "preceu.ph.qbr   %[" #TEMP2 "],   %[" #TEMP2 "]                       \n\t"  \
  "preceu.ph.qbr   %[" #TEMP3 "],   %[" #TEMP3 "]                       \n\t"  \
  "subq.ph         %[" #TEMP0 "],   %[" #TEMP0 "],   %[" #TEMP1 "]      \n\t"  \
  "subq.ph         %[" #TEMP2 "],   %[" #TEMP2 "],   %[" #TEMP3 "]      \n\t"  \
  "rotr            %[" #TEMP0 "],   %[" #TEMP0 "],   16                 \n\t"  \
  "addq.ph         %[" #TEMP1 "],   %[" #TEMP2 "],   %[" #TEMP0 "]      \n\t"  \
  "subq.ph         %[" #TEMP3 "],   %[" #TEMP2 "],   %[" #TEMP0 "]      \n\t"  \
  "seh             %[" #TEMP0 "],   %[" #TEMP1 "]                       \n\t"  \
  "sra             %[temp16],     %[" #TEMP1 "],   16                   \n\t"  \
  "seh             %[temp19],     %[" #TEMP3 "]                         \n\t"  \
  "sra             %[" #TEMP3 "],   %[" #TEMP3 "],   16                 \n\t"  \
  "subu            %[" #TEMP2 "],   %[" #TEMP0 "],   %[temp16]          \n\t"  \
  "addu            %[" #TEMP0 "],   %[" #TEMP0 "],   %[temp16]          \n\t"  \
  "mul             %[temp17],     %[temp19],     %[c2217]               \n\t"  \
  "mul             %[temp18],     %[" #TEMP3 "],   %[c5352]             \n\t"  \
  "mul             %[" #TEMP1 "],   %[temp19],     %[c5352]             \n\t"  \
  "mul             %[temp16],     %[" #TEMP3 "],   %[c2217]             \n\t"  \
  "sll             %[" #TEMP2 "],   %[" #TEMP2 "],   3                  \n\t"  \
  "sll             %[" #TEMP0 "],   %[" #TEMP0 "],   3                  \n\t"  \
  "subu            %[" #TEMP3 "],   %[temp17],     %[temp18]            \n\t"  \
  "addu            %[" #TEMP1 "],   %[temp16],     %[" #TEMP1 "]        \n\t"  \
  "addiu           %[" #TEMP3 "],   %[" #TEMP3 "],   937                \n\t"  \
  "addiu           %[" #TEMP1 "],   %[" #TEMP1 "],   1812               \n\t"  \
  "sra             %[" #TEMP3 "],   %[" #TEMP3 "],   9                  \n\t"  \
  "sra             %[" #TEMP1 "],   %[" #TEMP1 "],   9                  \n\t"

// macro for one vertical pass in FTransform
// temp0..temp15 holds tmp[0]..tmp[15]
// A..D - offsets in bytes to store to out buffer
// TEMP0, TEMP4, TEMP8 and TEMP12 - registers for corresponding tmp elements
#define VERTICAL_PASS(A, B, C, D, TEMP0, TEMP4, TEMP8, TEMP12)                 \
  "addu            %[temp16],     %[" #TEMP0 "],   %[" #TEMP12 "]   \n\t"      \
  "subu            %[temp19],     %[" #TEMP0 "],   %[" #TEMP12 "]   \n\t"      \
  "addu            %[temp17],     %[" #TEMP4 "],   %[" #TEMP8 "]    \n\t"      \
  "subu            %[temp18],     %[" #TEMP4 "],   %[" #TEMP8 "]    \n\t"      \
  "mul             %[" #TEMP8 "],   %[temp19],     %[c2217]         \n\t"      \
  "mul             %[" #TEMP12 "],  %[temp18],     %[c2217]         \n\t"      \
  "mul             %[" #TEMP4 "],   %[temp19],     %[c5352]         \n\t"      \
  "mul             %[temp18],     %[temp18],     %[c5352]           \n\t"      \
  "addiu           %[temp16],     %[temp16],     7                  \n\t"      \
  "addu            %[" #TEMP0 "],   %[temp16],     %[temp17]        \n\t"      \
  "sra             %[" #TEMP0 "],   %[" #TEMP0 "],   4              \n\t"      \
  "addu            %[" #TEMP12 "],  %[" #TEMP12 "],  %[" #TEMP4 "]  \n\t"      \
  "subu            %[" #TEMP4 "],   %[temp16],     %[temp17]        \n\t"      \
  "sra             %[" #TEMP4 "],   %[" #TEMP4 "],   4              \n\t"      \
  "addiu           %[" #TEMP8 "],   %[" #TEMP8 "],   30000          \n\t"      \
  "addiu           %[" #TEMP12 "],  %[" #TEMP12 "],  12000          \n\t"      \
  "addiu           %[" #TEMP8 "],   %[" #TEMP8 "],   21000          \n\t"      \
  "subu            %[" #TEMP8 "],   %[" #TEMP8 "],   %[temp18]      \n\t"      \
  "sra             %[" #TEMP12 "],  %[" #TEMP12 "],  16             \n\t"      \
  "sra             %[" #TEMP8 "],   %[" #TEMP8 "],   16             \n\t"      \
  "addiu           %[temp16],     %[" #TEMP12 "],  1                \n\t"      \
  "movn            %[" #TEMP12 "],  %[temp16],     %[temp19]        \n\t"      \
  "sh              %[" #TEMP0 "],   " #A "(%[temp20])               \n\t"      \
  "sh              %[" #TEMP4 "],   " #C "(%[temp20])               \n\t"      \
  "sh              %[" #TEMP8 "],   " #D "(%[temp20])               \n\t"      \
  "sh              %[" #TEMP12 "],  " #B "(%[temp20])               \n\t"

static void FTransform(const uint8_t* src, const uint8_t* ref, int16_t* out) {
  const int c2217 = 2217;
  const int c5352 = 5352;
  int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8;
  int temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16;
  int temp17, temp18, temp19, temp20;
  const int* const args[3] =
      { (const int*)src, (const int*)ref, (const int*)out };

  __asm__ volatile (
    HORIZONTAL_PASS(0, temp0,  temp1,  temp2,  temp3)
    HORIZONTAL_PASS(1, temp4,  temp5,  temp6,  temp7)
    HORIZONTAL_PASS(2, temp8,  temp9,  temp10, temp11)
    HORIZONTAL_PASS(3, temp12, temp13, temp14, temp15)
    "lw            %[temp20],     8(%[args])                  \n\t"
    VERTICAL_PASS(0,  8, 16, 24, temp0, temp4, temp8,  temp12)
    VERTICAL_PASS(2, 10, 18, 26, temp1, temp5, temp9,  temp13)
    VERTICAL_PASS(4, 12, 20, 28, temp2, temp6, temp10, temp14)
    VERTICAL_PASS(6, 14, 22, 30, temp3, temp7, temp11, temp15)
    OUTPUT_EARLY_CLOBBER_REGS_18(),
      [temp0]"=&r"(temp0), [temp19]"=&r"(temp19), [temp20]"=&r"(temp20)
    : [args]"r"(args), [c2217]"r"(c2217), [c5352]"r"(c5352)
    : "memory", "hi", "lo"
  );
}

#undef VERTICAL_PASS
#undef HORIZONTAL_PASS

static WEBP_INLINE void ITransformOne(const uint8_t* ref, const int16_t* in,
                                      uint8_t* dst) {
  int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9;
  int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17, temp18;

  __asm__ volatile (
    "ulw              %[temp1],   0(%[in])                 \n\t"
    "ulw              %[temp2],   16(%[in])                \n\t"
    LOAD_IN_X2(temp5, temp6, 24, 26)
    ADD_SUB_HALVES(temp3, temp4, temp1, temp2)
    LOAD_IN_X2(temp1, temp2, 8, 10)
    MUL_SHIFT_SUM(temp7, temp8, temp9, temp10, temp11, temp12, temp13, temp14,
                  temp10, temp8, temp9, temp7, temp1, temp2, temp5, temp6,
                  temp13, temp11, temp14, temp12)
    INSERT_HALF_X2(temp8, temp7, temp10, temp9)
    "ulw              %[temp17],  4(%[in])                 \n\t"
    "ulw              %[temp18],  20(%[in])                \n\t"
    ADD_SUB_HALVES(temp1, temp2, temp3, temp8)
    ADD_SUB_HALVES(temp5, temp6, temp4, temp7)
    ADD_SUB_HALVES(temp7, temp8, temp17, temp18)
    LOAD_IN_X2(temp17, temp18, 12, 14)
    LOAD_IN_X2(temp9, temp10, 28, 30)
    MUL_SHIFT_SUM(temp11, temp12, temp13, temp14, temp15, temp16, temp4, temp17,
                  temp12, temp14, temp11, temp13, temp17, temp18, temp9, temp10,
                  temp15, temp4, temp16, temp17)
    INSERT_HALF_X2(temp11, temp12, temp13, temp14)
    ADD_SUB_HALVES(temp17, temp8, temp8, temp11)
    ADD_SUB_HALVES(temp3, temp4, temp7, temp12)

    // horizontal
    SRA_16(temp9, temp10, temp11, temp12, temp1, temp2, temp5, temp6)
    INSERT_HALF_X2(temp1, temp6, temp5, temp2)
    SRA_16(temp13, temp14, temp15, temp16, temp3, temp4, temp17, temp8)
    "repl.ph          %[temp2],   0x4                      \n\t"
    INSERT_HALF_X2(temp3, temp8, temp17, temp4)
    "addq.ph          %[temp1],   %[temp1],  %[temp2]      \n\t"
    "addq.ph          %[temp6],   %[temp6],  %[temp2]      \n\t"
    ADD_SUB_HALVES(temp2, temp4, temp1, temp3)
    ADD_SUB_HALVES(temp5, temp7, temp6, temp8)
    MUL_SHIFT_SUM(temp1, temp3, temp6, temp8, temp9, temp13, temp17, temp18,
                  temp3, temp13, temp1, temp9, temp9, temp13, temp11, temp15,
                  temp6, temp17, temp8, temp18)
    MUL_SHIFT_SUM(temp6, temp8, temp18, temp17, temp11, temp15, temp12, temp16,
                  temp8, temp15, temp6, temp11, temp12, temp16, temp10, temp14,
                  temp18, temp12, temp17, temp16)
    INSERT_HALF_X2(temp1, temp3, temp9, temp13)
    INSERT_HALF_X2(temp6, temp8, temp11, temp15)
    SHIFT_R_SUM_X2(temp9, temp10, temp11, temp12, temp13, temp14, temp15,
                   temp16, temp2, temp4, temp5, temp7, temp3, temp1, temp8,
                   temp6)
    PACK_2_HALVES_TO_WORD(temp1, temp2, temp3, temp4, temp9, temp12, temp13,
                          temp16, temp11, temp10, temp15, temp14)
    LOAD_WITH_OFFSET_X4(temp10, temp11, temp14, temp15, ref,
                        0, 0, 0, 0,
                        0, 1, 2, 3,
                        BPS)
    CONVERT_2_BYTES_TO_HALF(temp5, temp6, temp7, temp8, temp17, temp18, temp10,
                            temp11, temp10, temp11, temp14, temp15)
    STORE_SAT_SUM_X2(temp5, temp6, temp7, temp8, temp17, temp18, temp10, temp11,
                     temp9, temp12, temp1, temp2, temp13, temp16, temp3, temp4,
                     dst, 0, 1, 2, 3, BPS)

    OUTPUT_EARLY_CLOBBER_REGS_18()
    : [dst]"r"(dst), [in]"r"(in), [kC1]"r"(kC1), [kC2]"r"(kC2), [ref]"r"(ref)
    : "memory", "hi", "lo"
  );
}

static void ITransform(const uint8_t* ref, const int16_t* in, uint8_t* dst,
                       int do_two) {
  ITransformOne(ref, in, dst);
  if (do_two) {
    ITransformOne(ref + 4, in + 16, dst + 4);
  }
}

static int Disto4x4(const uint8_t* const a, const uint8_t* const b,
                    const uint16_t* const w) {
  int temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8, temp9;
  int temp10, temp11, temp12, temp13, temp14, temp15, temp16, temp17;

  __asm__ volatile (
    LOAD_WITH_OFFSET_X4(temp1, temp2, temp3, temp4, a,
                        0, 0, 0, 0,
                        0, 1, 2, 3,
                        BPS)
    CONVERT_2_BYTES_TO_HALF(temp5, temp6, temp7, temp8, temp9,temp10, temp11,
                            temp12, temp1, temp2, temp3, temp4)
    ADD_SUB_HALVES_X4(temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8,
                      temp5, temp6, temp7, temp8, temp9, temp10, temp11, temp12)
    PACK_2_HALVES_TO_WORD(temp9, temp10, temp11, temp12, temp1, temp3, temp5,
                          temp7, temp2, temp4, temp6, temp8)
    ADD_SUB_HALVES_X4(temp2, temp4, temp6, temp8, temp9, temp1, temp3, temp10,
                      temp1, temp9, temp3, temp10, temp5, temp11, temp7, temp12)
    ADD_SUB_HALVES_X4(temp5, temp11, temp7, temp2, temp9, temp3, temp6, temp12,
                      temp2, temp9, temp6, temp3, temp4, temp1, temp8, temp10)
    ADD_SUB_HALVES_X4(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2,
                      temp5, temp7, temp11, temp2, temp9, temp6, temp3, temp12)
    ABS_X8(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2)
    LOAD_WITH_OFFSET_X4(temp3, temp6, temp9, temp12, w,
                        0, 4, 8, 12,
                        0, 0, 0, 0,
                        0)
    LOAD_WITH_OFFSET_X4(temp13, temp14, temp15, temp16, w,
                        0, 4, 8, 12,
                        1, 1, 1, 1,
                        16)
    MUL_HALF(temp17, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8,
             temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16)
    LOAD_WITH_OFFSET_X4(temp1, temp2, temp3, temp4, b,
                        0, 0, 0, 0,
                        0, 1, 2, 3,
                        BPS)
    CONVERT_2_BYTES_TO_HALF(temp5,temp6, temp7, temp8, temp9,temp10, temp11,
                            temp12, temp1, temp2, temp3, temp4)
    ADD_SUB_HALVES_X4(temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8,
                      temp5, temp6, temp7, temp8, temp9, temp10, temp11, temp12)
    PACK_2_HALVES_TO_WORD(temp9, temp10, temp11, temp12, temp1, temp3, temp5,
                          temp7, temp2, temp4, temp6, temp8)
    ADD_SUB_HALVES_X4(temp2, temp4, temp6, temp8, temp9, temp1, temp3, temp10,
                      temp1, temp9, temp3, temp10, temp5, temp11, temp7, temp12)
    ADD_SUB_HALVES_X4(temp5, temp11, temp7, temp2, temp9, temp3, temp6, temp12,
                      temp2, temp9, temp6, temp3, temp4, temp1, temp8, temp10)
    ADD_SUB_HALVES_X4(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2,
                      temp5, temp7, temp11, temp2, temp9, temp6, temp3, temp12)
    ABS_X8(temp1, temp4, temp10, temp8, temp7, temp11, temp5, temp2)
    LOAD_WITH_OFFSET_X4(temp3, temp6, temp9, temp12, w,
                        0, 4, 8, 12,
                        0, 0, 0, 0,
                        0)
    LOAD_WITH_OFFSET_X4(temp13, temp14, temp15, temp16, w,
                        0, 4, 8, 12,
                        1, 1, 1, 1,
                        16)
    MUL_HALF(temp3, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8,
             temp9, temp10, temp11, temp12, temp13, temp14, temp15, temp16)
    OUTPUT_EARLY_CLOBBER_REGS_17()
    : [a]"r"(a), [b]"r"(b), [w]"r"(w)
    : "memory", "hi", "lo"
  );
  return abs(temp3 - temp17) >> 5;
}

static int Disto16x16(const uint8_t* const a, const uint8_t* const b,
                      const uint16_t* const w) {
  int D = 0;
  int x, y;
  for (y = 0; y < 16 * BPS; y += 4 * BPS) {
    for (x = 0; x < 16; x += 4) {
      D += Disto4x4(a + x + y, b + x + y, w);
    }
  }
  return D;
}

//------------------------------------------------------------------------------
// Intra predictions

#define FILL_PART(J, SIZE)                                            \
    "usw        %[value],  0+" #J "*" XSTR(BPS) "(%[dst])  \n\t"      \
    "usw        %[value],  4+" #J "*" XSTR(BPS) "(%[dst])  \n\t"      \
  ".if " #SIZE " == 16                                     \n\t"      \
    "usw        %[value],  8+" #J "*" XSTR(BPS) "(%[dst])  \n\t"      \
    "usw        %[value], 12+" #J "*" XSTR(BPS) "(%[dst])  \n\t"      \
  ".endif                                                  \n\t"

#define FILL_8_OR_16(DST, VALUE, SIZE) do {                         \
  int value = (VALUE);                                              \
  __asm__ volatile (                                                \
    "replv.qb   %[value],  %[value]                      \n\t"      \
    FILL_PART( 0, SIZE)                                             \
    FILL_PART( 1, SIZE)                                             \
    FILL_PART( 2, SIZE)                                             \
    FILL_PART( 3, SIZE)                                             \
    FILL_PART( 4, SIZE)                                             \
    FILL_PART( 5, SIZE)                                             \
    FILL_PART( 6, SIZE)                                             \
    FILL_PART( 7, SIZE)                                             \
  ".if " #SIZE " == 16                                   \n\t"      \
    FILL_PART( 8, 16)                                               \
    FILL_PART( 9, 16)                                               \
    FILL_PART(10, 16)                                               \
    FILL_PART(11, 16)                                               \
    FILL_PART(12, 16)                                               \
    FILL_PART(13, 16)                                               \
    FILL_PART(14, 16)                                               \
    FILL_PART(15, 16)                                               \
  ".endif                                                \n\t"      \
    : [value]"+&r"(value)                                           \
    : [dst]"r"((DST))                                               \
    : "memory"                                                      \
  );                                                                \
} while (0)

#define VERTICAL_PRED(DST, TOP, SIZE)                                          \
static WEBP_INLINE void VerticalPred##SIZE(uint8_t* (DST),                     \
                                           const uint8_t* (TOP)) {             \
  int j;                                                                       \
  if ((TOP)) {                                                                 \
    for (j = 0; j < (SIZE); ++j) memcpy((DST) + j * BPS, (TOP), (SIZE));       \
  } else {                                                                     \
    FILL_8_OR_16((DST), 127, (SIZE));                                          \
  }                                                                            \
}

VERTICAL_PRED(dst, top, 8)
VERTICAL_PRED(dst, top, 16)

#undef VERTICAL_PRED

#define HORIZONTAL_PRED(DST, LEFT, SIZE)                                       \
static WEBP_INLINE void HorizontalPred##SIZE(uint8_t* (DST),                   \
                                             const uint8_t* (LEFT)) {          \
  if (LEFT) {                                                                  \
    int j;                                                                     \
    for (j = 0; j < (SIZE); ++j) {                                             \
      memset((DST) + j * BPS, (LEFT)[j], (SIZE));                              \
    }                                                                          \
  } else {                                                                     \
    FILL_8_OR_16((DST), 129, (SIZE));                                          \
  }                                                                            \
}

HORIZONTAL_PRED(dst, left, 8)
HORIZONTAL_PRED(dst, left, 16)

#undef HORIZONTAL_PRED

#define CLIPPING()                                                             \
  "preceu.ph.qbl   %[temp2],   %[temp0]                  \n\t"                 \
  "preceu.ph.qbr   %[temp0],   %[temp0]                  \n\t"                 \
  "preceu.ph.qbl   %[temp3],   %[temp1]                  \n\t"                 \
  "preceu.ph.qbr   %[temp1],   %[temp1]                  \n\t"                 \
  "addu.ph         %[temp2],   %[temp2],   %[leftY_1]    \n\t"                 \
  "addu.ph         %[temp0],   %[temp0],   %[leftY_1]    \n\t"                 \
  "addu.ph         %[temp3],   %[temp3],   %[leftY_1]    \n\t"                 \
  "addu.ph         %[temp1],   %[temp1],   %[leftY_1]    \n\t"                 \
  "shll_s.ph       %[temp2],   %[temp2],   7             \n\t"                 \
  "shll_s.ph       %[temp0],   %[temp0],   7             \n\t"                 \
  "shll_s.ph       %[temp3],   %[temp3],   7             \n\t"                 \
  "shll_s.ph       %[temp1],   %[temp1],   7             \n\t"                 \
  "precrqu_s.qb.ph %[temp0],   %[temp2],   %[temp0]      \n\t"                 \
  "precrqu_s.qb.ph %[temp1],   %[temp3],   %[temp1]      \n\t"

#define CLIP_8B_TO_DST(DST, LEFT, TOP, SIZE) do {                              \
  int leftY_1 = ((int)(LEFT)[y] << 16) + (LEFT)[y];                            \
  int temp0, temp1, temp2, temp3;                                              \
  __asm__ volatile (                                                           \
    "replv.ph        %[leftY_1], %[leftY_1]              \n\t"                 \
    "ulw             %[temp0],   0(%[top])               \n\t"                 \
    "ulw             %[temp1],   4(%[top])               \n\t"                 \
    "subu.ph         %[leftY_1], %[leftY_1], %[left_1]   \n\t"                 \
    CLIPPING()                                                                 \
    "usw             %[temp0],   0(%[dst])               \n\t"                 \
    "usw             %[temp1],   4(%[dst])               \n\t"                 \
  ".if " #SIZE " == 16                                   \n\t"                 \
    "ulw             %[temp0],   8(%[top])               \n\t"                 \
    "ulw             %[temp1],   12(%[top])              \n\t"                 \
    CLIPPING()                                                                 \
    "usw             %[temp0],   8(%[dst])               \n\t"                 \
    "usw             %[temp1],   12(%[dst])              \n\t"                 \
  ".endif                                                \n\t"                 \
    : [leftY_1]"+&r"(leftY_1), [temp0]"=&r"(temp0), [temp1]"=&r"(temp1),       \
      [temp2]"=&r"(temp2), [temp3]"=&r"(temp3)                                 \
    : [left_1]"r"(left_1), [top]"r"((TOP)), [dst]"r"((DST))                    \
    : "memory"                                                                 \
  );                                                                           \
} while (0)

#define CLIP_TO_DST(DST, LEFT, TOP, SIZE) do {                                 \
  int y;                                                                       \
  const int left_1 = ((int)(LEFT)[-1] << 16) + (LEFT)[-1];                     \
  for (y = 0; y < (SIZE); ++y) {                                               \
    CLIP_8B_TO_DST((DST), (LEFT), (TOP), (SIZE));                              \
    (DST) += BPS;                                                              \
  }                                                                            \
} while (0)

#define TRUE_MOTION(DST, LEFT, TOP, SIZE)                                      \
static WEBP_INLINE void TrueMotion##SIZE(uint8_t* (DST), const uint8_t* (LEFT),\
                                         const uint8_t* (TOP)) {               \
  if ((LEFT) != NULL) {                                                        \
    if ((TOP) != NULL) {                                                       \
      CLIP_TO_DST((DST), (LEFT), (TOP), (SIZE));                               \
    } else {                                                                   \
      HorizontalPred##SIZE((DST), (LEFT));                                     \
    }                                                                          \
  } else {                                                                     \
    /* true motion without left samples (hence: with default 129 value)    */  \
    /* is equivalent to VE prediction where you just copy the top samples. */  \
    /* Note that if top samples are not available, the default value is    */  \
    /* then 129, and not 127 as in the VerticalPred case.                  */  \
    if ((TOP) != NULL) {                                                       \
      VerticalPred##SIZE((DST), (TOP));                                        \
    } else {                                                                   \
      FILL_8_OR_16((DST), 129, (SIZE));                                        \
    }                                                                          \
  }                                                                            \
}

TRUE_MOTION(dst, left, top, 8)
TRUE_MOTION(dst, left, top, 16)

#undef TRUE_MOTION
#undef CLIP_TO_DST
#undef CLIP_8B_TO_DST
#undef CLIPPING

static WEBP_INLINE void DCMode16(uint8_t* dst, const uint8_t* left,
                                 const uint8_t* top) {
  int DC, DC1;
  int temp0, temp1, temp2, temp3;

  __asm__ volatile(
    "beqz        %[top],   2f                  \n\t"
    LOAD_WITH_OFFSET_X4(temp0, temp1, temp2, temp3, top,
                        0, 4, 8, 12,
                        0, 0, 0, 0,
                        0)
    "raddu.w.qb  %[temp0], %[temp0]            \n\t"
    "raddu.w.qb  %[temp1], %[temp1]            \n\t"
    "raddu.w.qb  %[temp2], %[temp2]            \n\t"
    "raddu.w.qb  %[temp3], %[temp3]            \n\t"
    "addu        %[temp0], %[temp0], %[temp1]  \n\t"
    "addu        %[temp2], %[temp2], %[temp3]  \n\t"
    "addu        %[DC],    %[temp0], %[temp2]  \n\t"
    "move        %[DC1],   %[DC]               \n\t"
    "beqz        %[left],  1f                  \n\t"
    LOAD_WITH_OFFSET_X4(temp0, temp1, temp2, temp3, left,
                        0, 4, 8, 12,
                        0, 0, 0, 0,
                        0)
    "raddu.w.qb  %[temp0], %[temp0]            \n\t"
    "raddu.w.qb  %[temp1], %[temp1]            \n\t"
    "raddu.w.qb  %[temp2], %[temp2]            \n\t"
    "raddu.w.qb  %[temp3], %[temp3]            \n\t"
    "addu        %[temp0], %[temp0], %[temp1]  \n\t"
    "addu        %[temp2], %[temp2], %[temp3]  \n\t"
    "addu        %[DC1],   %[temp0], %[temp2]  \n\t"
  "1:                                          \n\t"
    "addu        %[DC],   %[DC],     %[DC1]    \n\t"
    "j           3f                            \n\t"
  "2:                                          \n\t"
    "beqz        %[left],  4f                  \n\t"
    LOAD_WITH_OFFSET_X4(temp0, temp1, temp2, temp3, left,
                        0, 4, 8, 12,
                        0, 0, 0, 0,
                        0)
    "raddu.w.qb  %[temp0], %[temp0]            \n\t"
    "raddu.w.qb  %[temp1], %[temp1]            \n\t"
    "raddu.w.qb  %[temp2], %[temp2]            \n\t"
    "raddu.w.qb  %[temp3], %[temp3]            \n\t"
    "addu        %[temp0], %[temp0], %[temp1]  \n\t"
    "addu        %[temp2], %[temp2], %[temp3]  \n\t"
    "addu        %[DC],    %[temp0], %[temp2]  \n\t"
    "addu        %[DC],    %[DC],    %[DC]     \n\t"
  "3:                                          \n\t"
    "shra_r.w    %[DC],    %[DC],    5         \n\t"
    "j           5f                            \n\t"
  "4:                                          \n\t"
    "li          %[DC],    0x80                \n\t"
  "5:                                          \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [DC]"=&r"(DC),
      [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [DC1]"=&r"(DC1)
    : [left]"r"(left), [top]"r"(top)
    : "memory"
  );

  FILL_8_OR_16(dst, DC, 16);
}

static WEBP_INLINE void DCMode8(uint8_t* dst, const uint8_t* left,
                                const uint8_t* top) {
  int DC, DC1;
  int temp0, temp1, temp2, temp3;

  __asm__ volatile(
    "beqz        %[top],   2f                  \n\t"
    "ulw         %[temp0], 0(%[top])           \n\t"
    "ulw         %[temp1], 4(%[top])           \n\t"
    "raddu.w.qb  %[temp0], %[temp0]            \n\t"
    "raddu.w.qb  %[temp1], %[temp1]            \n\t"
    "addu        %[DC],    %[temp0], %[temp1]  \n\t"
    "move        %[DC1],   %[DC]               \n\t"
    "beqz        %[left],  1f                  \n\t"
    "ulw         %[temp2], 0(%[left])          \n\t"
    "ulw         %[temp3], 4(%[left])          \n\t"
    "raddu.w.qb  %[temp2], %[temp2]            \n\t"
    "raddu.w.qb  %[temp3], %[temp3]            \n\t"
    "addu        %[DC1],   %[temp2], %[temp3]  \n\t"
  "1:                                          \n\t"
    "addu        %[DC],    %[DC],    %[DC1]    \n\t"
    "j           3f                            \n\t"
  "2:                                          \n\t"
    "beqz        %[left],  4f                  \n\t"
    "ulw         %[temp2], 0(%[left])          \n\t"
    "ulw         %[temp3], 4(%[left])          \n\t"
    "raddu.w.qb  %[temp2], %[temp2]            \n\t"
    "raddu.w.qb  %[temp3], %[temp3]            \n\t"
    "addu        %[DC],    %[temp2], %[temp3]  \n\t"
    "addu        %[DC],    %[DC],    %[DC]     \n\t"
  "3:                                          \n\t"
    "shra_r.w    %[DC], %[DC], 4               \n\t"
    "j           5f                            \n\t"
  "4:                                          \n\t"
    "li          %[DC], 0x80                   \n\t"
  "5:                                          \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [DC]"=&r"(DC),
      [temp2]"=&r"(temp2), [temp3]"=&r"(temp3), [DC1]"=&r"(DC1)
    : [left]"r"(left), [top]"r"(top)
    : "memory"
  );

  FILL_8_OR_16(dst, DC, 8);
}

static void DC4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1;
  __asm__ volatile(
    "ulw          %[temp0],   0(%[top])               \n\t"
    "ulw          %[temp1],   -5(%[top])              \n\t"
    "raddu.w.qb   %[temp0],   %[temp0]                \n\t"
    "raddu.w.qb   %[temp1],   %[temp1]                \n\t"
    "addu         %[temp0],   %[temp0],    %[temp1]   \n\t"
    "addiu        %[temp0],   %[temp0],    4          \n\t"
    "srl          %[temp0],   %[temp0],    3          \n\t"
    "replv.qb     %[temp0],   %[temp0]                \n\t"
    "usw          %[temp0],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "usw          %[temp0],   1*" XSTR(BPS) "(%[dst]) \n\t"
    "usw          %[temp0],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "usw          %[temp0],   3*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void TM4(uint8_t* dst, const uint8_t* top) {
  int a10, a32, temp0, temp1, temp2, temp3, temp4, temp5;
  const int c35 = 0xff00ff;
  __asm__ volatile (
    "lbu              %[temp1],  0(%[top])                     \n\t"
    "lbu              %[a10],    1(%[top])                     \n\t"
    "lbu              %[temp2],  2(%[top])                     \n\t"
    "lbu              %[a32],    3(%[top])                     \n\t"
    "ulw              %[temp0],  -5(%[top])                    \n\t"
    "lbu              %[temp4],  -1(%[top])                    \n\t"
    "append           %[a10],    %[temp1],   16                \n\t"
    "append           %[a32],    %[temp2],   16                \n\t"
    "replv.ph         %[temp4],  %[temp4]                      \n\t"
    "shrl.ph          %[temp1],  %[temp0],   8                 \n\t"
    "and              %[temp0],  %[temp0],   %[c35]            \n\t"
    "subu.ph          %[temp1],  %[temp1],   %[temp4]          \n\t"
    "subu.ph          %[temp0],  %[temp0],   %[temp4]          \n\t"
    "srl              %[temp2],  %[temp1],   16                \n\t"
    "srl              %[temp3],  %[temp0],   16                \n\t"
    "replv.ph         %[temp2],  %[temp2]                      \n\t"
    "replv.ph         %[temp3],  %[temp3]                      \n\t"
    "replv.ph         %[temp4],  %[temp1]                      \n\t"
    "replv.ph         %[temp5],  %[temp0]                      \n\t"
    "addu.ph          %[temp0],  %[temp3],   %[a10]            \n\t"
    "addu.ph          %[temp1],  %[temp3],   %[a32]            \n\t"
    "addu.ph          %[temp3],  %[temp2],   %[a10]            \n\t"
    "addu.ph          %[temp2],  %[temp2],   %[a32]            \n\t"
    "shll_s.ph        %[temp0],  %[temp0],   7                 \n\t"
    "shll_s.ph        %[temp1],  %[temp1],   7                 \n\t"
    "shll_s.ph        %[temp3],  %[temp3],   7                 \n\t"
    "shll_s.ph        %[temp2],  %[temp2],   7                 \n\t"
    "precrqu_s.qb.ph  %[temp0],  %[temp1],   %[temp0]          \n\t"
    "precrqu_s.qb.ph  %[temp1],  %[temp2],   %[temp3]          \n\t"
    "addu.ph          %[temp2],  %[temp5],   %[a10]            \n\t"
    "addu.ph          %[temp3],  %[temp5],   %[a32]            \n\t"
    "addu.ph          %[temp5],  %[temp4],   %[a10]            \n\t"
    "addu.ph          %[temp4],  %[temp4],   %[a32]            \n\t"
    "shll_s.ph        %[temp2],  %[temp2],   7                 \n\t"
    "shll_s.ph        %[temp3],  %[temp3],   7                 \n\t"
    "shll_s.ph        %[temp4],  %[temp4],   7                 \n\t"
    "shll_s.ph        %[temp5],  %[temp5],   7                 \n\t"
    "precrqu_s.qb.ph  %[temp2],  %[temp3],   %[temp2]          \n\t"
    "precrqu_s.qb.ph  %[temp3],  %[temp4],   %[temp5]          \n\t"
    "usw              %[temp1],  0*" XSTR(BPS) "(%[dst])       \n\t"
    "usw              %[temp0],  1*" XSTR(BPS) "(%[dst])       \n\t"
    "usw              %[temp3],  2*" XSTR(BPS) "(%[dst])       \n\t"
    "usw              %[temp2],  3*" XSTR(BPS) "(%[dst])       \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [a10]"=&r"(a10), [a32]"=&r"(a32)
    : [c35]"r"(c35), [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void VE4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4, temp5, temp6;
  __asm__ volatile(
    "ulw             %[temp0],   -1(%[top])              \n\t"
    "ulh             %[temp1],   3(%[top])               \n\t"
    "preceu.ph.qbr   %[temp2],   %[temp0]                \n\t"
    "preceu.ph.qbl   %[temp3],   %[temp0]                \n\t"
    "preceu.ph.qbr   %[temp4],   %[temp1]                \n\t"
    "packrl.ph       %[temp5],   %[temp3],    %[temp2]   \n\t"
    "packrl.ph       %[temp6],   %[temp4],    %[temp3]   \n\t"
    "shll.ph         %[temp5],   %[temp5],    1          \n\t"
    "shll.ph         %[temp6],   %[temp6],    1          \n\t"
    "addq.ph         %[temp2],   %[temp5],    %[temp2]   \n\t"
    "addq.ph         %[temp6],   %[temp6],    %[temp4]   \n\t"
    "addq.ph         %[temp2],   %[temp2],    %[temp3]   \n\t"
    "addq.ph         %[temp6],   %[temp6],    %[temp3]   \n\t"
    "shra_r.ph       %[temp2],   %[temp2],    2          \n\t"
    "shra_r.ph       %[temp6],   %[temp6],    2          \n\t"
    "precr.qb.ph     %[temp4],   %[temp6],    %[temp2]   \n\t"
    "usw             %[temp4],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp4],   1*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp4],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp4],   3*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void HE4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4, temp5, temp6;
  __asm__ volatile(
    "ulw             %[temp0],   -4(%[top])              \n\t"
    "lbu             %[temp1],   -5(%[top])              \n\t"
    "preceu.ph.qbr   %[temp2],   %[temp0]                \n\t"
    "preceu.ph.qbl   %[temp3],   %[temp0]                \n\t"
    "replv.ph        %[temp4],   %[temp1]                \n\t"
    "packrl.ph       %[temp5],   %[temp3],    %[temp2]   \n\t"
    "packrl.ph       %[temp6],   %[temp2],    %[temp4]   \n\t"
    "shll.ph         %[temp5],   %[temp5],    1          \n\t"
    "shll.ph         %[temp6],   %[temp6],    1          \n\t"
    "addq.ph         %[temp3],   %[temp3],    %[temp5]   \n\t"
    "addq.ph         %[temp3],   %[temp3],    %[temp2]   \n\t"
    "addq.ph         %[temp2],   %[temp2],    %[temp6]   \n\t"
    "addq.ph         %[temp2],   %[temp2],    %[temp4]   \n\t"
    "shra_r.ph       %[temp3],   %[temp3],    2          \n\t"
    "shra_r.ph       %[temp2],   %[temp2],    2          \n\t"
    "replv.qb        %[temp0],   %[temp3]                \n\t"
    "replv.qb        %[temp1],   %[temp2]                \n\t"
    "srl             %[temp3],   %[temp3],    16         \n\t"
    "srl             %[temp2],   %[temp2],    16         \n\t"
    "replv.qb        %[temp3],   %[temp3]                \n\t"
    "replv.qb        %[temp2],   %[temp2]                \n\t"
    "usw             %[temp3],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp0],   1*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp2],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp1],   3*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void RD4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4, temp5;
  int temp6, temp7, temp8, temp9, temp10, temp11;
  __asm__ volatile(
    "ulw             %[temp0],    -5(%[top])               \n\t"
    "ulw             %[temp1],    -1(%[top])               \n\t"
    "preceu.ph.qbl   %[temp2],    %[temp0]                 \n\t"
    "preceu.ph.qbr   %[temp3],    %[temp0]                 \n\t"
    "preceu.ph.qbr   %[temp4],    %[temp1]                 \n\t"
    "preceu.ph.qbl   %[temp5],    %[temp1]                 \n\t"
    "packrl.ph       %[temp6],    %[temp2],    %[temp3]    \n\t"
    "packrl.ph       %[temp7],    %[temp4],    %[temp2]    \n\t"
    "packrl.ph       %[temp8],    %[temp5],    %[temp4]    \n\t"
    "shll.ph         %[temp6],    %[temp6],    1           \n\t"
    "addq.ph         %[temp9],    %[temp2],    %[temp6]    \n\t"
    "shll.ph         %[temp7],    %[temp7],    1           \n\t"
    "addq.ph         %[temp9],    %[temp9],    %[temp3]    \n\t"
    "shll.ph         %[temp8],    %[temp8],    1           \n\t"
    "shra_r.ph       %[temp9],    %[temp9],    2           \n\t"
    "addq.ph         %[temp10],   %[temp4],    %[temp7]    \n\t"
    "addq.ph         %[temp11],   %[temp5],    %[temp8]    \n\t"
    "addq.ph         %[temp10],   %[temp10],   %[temp2]    \n\t"
    "addq.ph         %[temp11],   %[temp11],   %[temp4]    \n\t"
    "shra_r.ph       %[temp10],   %[temp10],   2           \n\t"
    "shra_r.ph       %[temp11],   %[temp11],   2           \n\t"
    "lbu             %[temp0],    3(%[top])                \n\t"
    "lbu             %[temp1],    2(%[top])                \n\t"
    "lbu             %[temp2],    1(%[top])                \n\t"
    "sll             %[temp1],    %[temp1],    1           \n\t"
    "addu            %[temp0],    %[temp0],    %[temp1]    \n\t"
    "addu            %[temp0],    %[temp0],    %[temp2]    \n\t"
    "precr.qb.ph     %[temp9],    %[temp10],   %[temp9]    \n\t"
    "shra_r.w        %[temp0],    %[temp0],    2           \n\t"
    "precr.qb.ph     %[temp10],   %[temp11],   %[temp10]   \n\t"
    "usw             %[temp9],    3*" XSTR(BPS) "(%[dst])  \n\t"
    "usw             %[temp10],   1*" XSTR(BPS) "(%[dst])  \n\t"
    "prepend         %[temp9],    %[temp11],   8           \n\t"
    "prepend         %[temp10],   %[temp0],    8           \n\t"
    "usw             %[temp9],    2*" XSTR(BPS) "(%[dst])  \n\t"
    "usw             %[temp10],   0*" XSTR(BPS) "(%[dst])  \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
      [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void VR4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4;
  int temp5, temp6, temp7, temp8, temp9;
  __asm__ volatile (
    "ulw              %[temp0],   -4(%[top])              \n\t"
    "ulw              %[temp1],   0(%[top])               \n\t"
    "preceu.ph.qbl    %[temp2],   %[temp0]                \n\t"
    "preceu.ph.qbr    %[temp0],   %[temp0]                \n\t"
    "preceu.ph.qbla   %[temp3],   %[temp1]                \n\t"
    "preceu.ph.qbra   %[temp1],   %[temp1]                \n\t"
    "packrl.ph        %[temp7],   %[temp3],    %[temp2]   \n\t"
    "addqh_r.ph       %[temp4],   %[temp1],    %[temp3]   \n\t"
    "move             %[temp6],   %[temp1]                \n\t"
    "append           %[temp1],   %[temp2],    16         \n\t"
    "shll.ph          %[temp9],   %[temp6],    1          \n\t"
    "addqh_r.ph       %[temp5],   %[temp7],    %[temp6]   \n\t"
    "shll.ph          %[temp8],   %[temp7],    1          \n\t"
    "addu.ph          %[temp3],   %[temp7],    %[temp3]   \n\t"
    "addu.ph          %[temp1],   %[temp1],    %[temp6]   \n\t"
    "packrl.ph        %[temp7],   %[temp2],    %[temp0]   \n\t"
    "addu.ph          %[temp6],   %[temp0],    %[temp2]   \n\t"
    "addu.ph          %[temp3],   %[temp3],    %[temp9]   \n\t"
    "addu.ph          %[temp1],   %[temp1],    %[temp8]   \n\t"
    "shll.ph          %[temp7],   %[temp7],    1          \n\t"
    "shra_r.ph        %[temp3],   %[temp3],    2          \n\t"
    "shra_r.ph        %[temp1],   %[temp1],    2          \n\t"
    "addu.ph          %[temp6],   %[temp6],    %[temp7]   \n\t"
    "shra_r.ph        %[temp6],   %[temp6],    2          \n\t"
    "precrq.ph.w      %[temp8],   %[temp4],    %[temp5]   \n\t"
    "append           %[temp4],   %[temp5],    16         \n\t"
    "precrq.ph.w      %[temp2],   %[temp3],    %[temp1]   \n\t"
    "append           %[temp3],   %[temp1],    16         \n\t"
    "precr.qb.ph      %[temp8],   %[temp8],    %[temp4]   \n\t"
    "precr.qb.ph      %[temp3],   %[temp2],    %[temp3]   \n\t"
    "usw              %[temp8],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "usw              %[temp3],   1*" XSTR(BPS) "(%[dst]) \n\t"
    "append           %[temp3],   %[temp6],    8          \n\t"
    "srl              %[temp6],   %[temp6],    16         \n\t"
    "append           %[temp8],   %[temp6],    8          \n\t"
    "usw              %[temp3],   3*" XSTR(BPS) "(%[dst]) \n\t"
    "usw              %[temp8],   2*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
      [temp9]"=&r"(temp9)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void LD4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4, temp5;
  int temp6, temp7, temp8, temp9, temp10, temp11;
  __asm__ volatile(
    "ulw             %[temp0],    0(%[top])               \n\t"
    "ulw             %[temp1],    4(%[top])               \n\t"
    "preceu.ph.qbl   %[temp2],    %[temp0]                \n\t"
    "preceu.ph.qbr   %[temp3],    %[temp0]                \n\t"
    "preceu.ph.qbr   %[temp4],    %[temp1]                \n\t"
    "preceu.ph.qbl   %[temp5],    %[temp1]                \n\t"
    "packrl.ph       %[temp6],    %[temp2],    %[temp3]   \n\t"
    "packrl.ph       %[temp7],    %[temp4],    %[temp2]   \n\t"
    "packrl.ph       %[temp8],    %[temp5],    %[temp4]   \n\t"
    "shll.ph         %[temp6],    %[temp6],    1          \n\t"
    "addq.ph         %[temp9],    %[temp2],    %[temp6]   \n\t"
    "shll.ph         %[temp7],    %[temp7],    1          \n\t"
    "addq.ph         %[temp9],    %[temp9],    %[temp3]   \n\t"
    "shll.ph         %[temp8],    %[temp8],    1          \n\t"
    "shra_r.ph       %[temp9],    %[temp9],    2          \n\t"
    "addq.ph         %[temp10],   %[temp4],    %[temp7]   \n\t"
    "addq.ph         %[temp11],   %[temp5],    %[temp8]   \n\t"
    "addq.ph         %[temp10],   %[temp10],   %[temp2]   \n\t"
    "addq.ph         %[temp11],   %[temp11],   %[temp4]   \n\t"
    "shra_r.ph       %[temp10],   %[temp10],   2          \n\t"
    "shra_r.ph       %[temp11],   %[temp11],   2          \n\t"
    "srl             %[temp1],    %[temp1],    24         \n\t"
    "sll             %[temp1],    %[temp1],    1          \n\t"
    "raddu.w.qb      %[temp5],    %[temp5]                \n\t"
    "precr.qb.ph     %[temp9],    %[temp10],   %[temp9]   \n\t"
    "precr.qb.ph     %[temp10],   %[temp11],   %[temp10]  \n\t"
    "addu            %[temp1],    %[temp1],    %[temp5]   \n\t"
    "shra_r.w        %[temp1],    %[temp1],    2          \n\t"
    "usw             %[temp9],    0*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp10],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "prepend         %[temp9],    %[temp11],   8          \n\t"
    "prepend         %[temp10],   %[temp1],    8          \n\t"
    "usw             %[temp9],    1*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp10],   3*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
      [temp9]"=&r"(temp9), [temp10]"=&r"(temp10), [temp11]"=&r"(temp11)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void VL4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4;
  int temp5, temp6, temp7, temp8, temp9;
  __asm__ volatile (
    "ulw              %[temp0],   0(%[top])               \n\t"
    "ulw              %[temp1],   4(%[top])               \n\t"
    "preceu.ph.qbla   %[temp2],   %[temp0]                \n\t"
    "preceu.ph.qbra   %[temp0],   %[temp0]                \n\t"
    "preceu.ph.qbl    %[temp3],   %[temp1]                \n\t"
    "preceu.ph.qbr    %[temp1],   %[temp1]                \n\t"
    "addqh_r.ph       %[temp4],   %[temp0],    %[temp2]   \n\t"
    "packrl.ph        %[temp7],   %[temp1],    %[temp0]   \n\t"
    "precrq.ph.w      %[temp6],   %[temp1],    %[temp2]   \n\t"
    "shll.ph          %[temp9],   %[temp2],    1          \n\t"
    "addqh_r.ph       %[temp5],   %[temp7],    %[temp2]   \n\t"
    "shll.ph          %[temp8],   %[temp7],    1          \n\t"
    "addu.ph          %[temp2],   %[temp2],    %[temp6]   \n\t"
    "addu.ph          %[temp0],   %[temp0],    %[temp7]   \n\t"
    "packrl.ph        %[temp7],   %[temp3],    %[temp1]   \n\t"
    "addu.ph          %[temp6],   %[temp1],    %[temp3]   \n\t"
    "addu.ph          %[temp2],   %[temp2],    %[temp8]   \n\t"
    "addu.ph          %[temp0],   %[temp0],    %[temp9]   \n\t"
    "shll.ph          %[temp7],   %[temp7],    1          \n\t"
    "shra_r.ph        %[temp2],   %[temp2],    2          \n\t"
    "shra_r.ph        %[temp0],   %[temp0],    2          \n\t"
    "addu.ph          %[temp6],   %[temp6],    %[temp7]   \n\t"
    "shra_r.ph        %[temp6],   %[temp6],    2          \n\t"
    "precrq.ph.w      %[temp8],   %[temp5],    %[temp4]   \n\t"
    "append           %[temp5],   %[temp4],    16         \n\t"
    "precrq.ph.w      %[temp3],   %[temp2],    %[temp0]   \n\t"
    "append           %[temp2],   %[temp0],    16         \n\t"
    "precr.qb.ph      %[temp8],   %[temp8],    %[temp5]   \n\t"
    "precr.qb.ph      %[temp3],   %[temp3],    %[temp2]   \n\t"
    "usw              %[temp8],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "prepend          %[temp8],   %[temp6],    8          \n\t"
    "usw              %[temp3],   1*" XSTR(BPS) "(%[dst]) \n\t"
    "srl              %[temp6],   %[temp6],    16         \n\t"
    "prepend          %[temp3],   %[temp6],    8          \n\t"
    "usw              %[temp8],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "usw              %[temp3],   3*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
      [temp9]"=&r"(temp9)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void HD4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4;
  int temp5, temp6, temp7, temp8, temp9;
  __asm__ volatile (
    "ulw              %[temp0],   -5(%[top])              \n\t"
    "ulw              %[temp1],   -1(%[top])              \n\t"
    "preceu.ph.qbla   %[temp2],   %[temp0]                \n\t"
    "preceu.ph.qbra   %[temp0],   %[temp0]                \n\t"
    "preceu.ph.qbl    %[temp3],   %[temp1]                \n\t"
    "preceu.ph.qbr    %[temp1],   %[temp1]                \n\t"
    "addqh_r.ph       %[temp4],   %[temp0],    %[temp2]   \n\t"
    "packrl.ph        %[temp7],   %[temp1],    %[temp0]   \n\t"
    "precrq.ph.w      %[temp6],   %[temp1],    %[temp2]   \n\t"
    "shll.ph          %[temp9],   %[temp2],    1          \n\t"
    "addqh_r.ph       %[temp5],   %[temp7],    %[temp2]   \n\t"
    "shll.ph          %[temp8],   %[temp7],    1          \n\t"
    "addu.ph          %[temp2],   %[temp2],    %[temp6]   \n\t"
    "addu.ph          %[temp0],   %[temp0],    %[temp7]   \n\t"
    "packrl.ph        %[temp7],   %[temp3],    %[temp1]   \n\t"
    "addu.ph          %[temp6],   %[temp1],    %[temp3]   \n\t"
    "addu.ph          %[temp2],   %[temp2],    %[temp8]   \n\t"
    "addu.ph          %[temp0],   %[temp0],    %[temp9]   \n\t"
    "shll.ph          %[temp7],   %[temp7],    1          \n\t"
    "shra_r.ph        %[temp2],   %[temp2],    2          \n\t"
    "shra_r.ph        %[temp0],   %[temp0],    2          \n\t"
    "addu.ph          %[temp6],   %[temp6],    %[temp7]   \n\t"
    "shra_r.ph        %[temp6],   %[temp6],    2          \n\t"
    "precrq.ph.w      %[temp1],   %[temp2],    %[temp5]   \n\t"
    "precrq.ph.w      %[temp3],   %[temp0],    %[temp4]   \n\t"
    "precr.qb.ph      %[temp7],   %[temp6],    %[temp1]   \n\t"
    "precr.qb.ph      %[temp6],   %[temp1],    %[temp3]   \n\t"
    "usw              %[temp7],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "usw              %[temp6],   1*" XSTR(BPS) "(%[dst]) \n\t"
    "append           %[temp2],   %[temp5],    16         \n\t"
    "append           %[temp0],   %[temp4],    16         \n\t"
    "precr.qb.ph      %[temp5],   %[temp3],    %[temp2]   \n\t"
    "precr.qb.ph      %[temp4],   %[temp2],    %[temp0]   \n\t"
    "usw              %[temp5],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "usw              %[temp4],   3*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
      [temp9]"=&r"(temp9)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

static void HU4(uint8_t* dst, const uint8_t* top) {
  int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7;
  __asm__ volatile (
    "ulw             %[temp0],   -5(%[top])              \n\t"
    "preceu.ph.qbl   %[temp1],   %[temp0]                \n\t"
    "preceu.ph.qbr   %[temp2],   %[temp0]                \n\t"
    "packrl.ph       %[temp3],   %[temp1],    %[temp2]   \n\t"
    "replv.qb        %[temp7],   %[temp2]                \n\t"
    "addqh_r.ph      %[temp4],   %[temp1],    %[temp3]   \n\t"
    "addqh_r.ph      %[temp5],   %[temp3],    %[temp2]   \n\t"
    "shll.ph         %[temp6],   %[temp3],    1          \n\t"
    "addu.ph         %[temp3],   %[temp2],    %[temp3]   \n\t"
    "addu.ph         %[temp6],   %[temp1],    %[temp6]   \n\t"
    "shll.ph         %[temp0],   %[temp2],    1          \n\t"
    "addu.ph         %[temp6],   %[temp6],    %[temp2]   \n\t"
    "addu.ph         %[temp0],   %[temp3],    %[temp0]   \n\t"
    "shra_r.ph       %[temp6],   %[temp6],    2          \n\t"
    "shra_r.ph       %[temp0],   %[temp0],    2          \n\t"
    "packrl.ph       %[temp3],   %[temp6],    %[temp5]   \n\t"
    "precrq.ph.w     %[temp2],   %[temp6],    %[temp4]   \n\t"
    "append          %[temp0],   %[temp5],    16         \n\t"
    "precr.qb.ph     %[temp3],   %[temp3],    %[temp2]   \n\t"
    "usw             %[temp3],   0*" XSTR(BPS) "(%[dst]) \n\t"
    "precr.qb.ph     %[temp1],   %[temp7],    %[temp0]   \n\t"
    "usw             %[temp7],   3*" XSTR(BPS) "(%[dst]) \n\t"
    "packrl.ph       %[temp2],   %[temp1],    %[temp3]   \n\t"
    "usw             %[temp1],   2*" XSTR(BPS) "(%[dst]) \n\t"
    "usw             %[temp2],   1*" XSTR(BPS) "(%[dst]) \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7)
    : [top]"r"(top), [dst]"r"(dst)
    : "memory"
  );
}

//------------------------------------------------------------------------------
// Chroma 8x8 prediction (paragraph 12.2)

static void IntraChromaPreds(uint8_t* dst, const uint8_t* left,
                             const uint8_t* top) {
  // U block
  DCMode8(C8DC8 + dst, left, top);
  VerticalPred8(C8VE8 + dst, top);
  HorizontalPred8(C8HE8 + dst, left);
  TrueMotion8(C8TM8 + dst, left, top);
  // V block
  dst += 8;
  if (top) top += 8;
  if (left) left += 16;
  DCMode8(C8DC8 + dst, left, top);
  VerticalPred8(C8VE8 + dst, top);
  HorizontalPred8(C8HE8 + dst, left);
  TrueMotion8(C8TM8 + dst, left, top);
}

//------------------------------------------------------------------------------
// luma 16x16 prediction (paragraph 12.3)

static void Intra16Preds(uint8_t* dst,
                         const uint8_t* left, const uint8_t* top) {
  DCMode16(I16DC16 + dst, left, top);
  VerticalPred16(I16VE16 + dst, top);
  HorizontalPred16(I16HE16 + dst, left);
  TrueMotion16(I16TM16 + dst, left, top);
}

// Left samples are top[-5 .. -2], top_left is top[-1], top are
// located at top[0..3], and top right is top[4..7]
static void Intra4Preds(uint8_t* dst, const uint8_t* top) {
  DC4(I4DC4 + dst, top);
  TM4(I4TM4 + dst, top);
  VE4(I4VE4 + dst, top);
  HE4(I4HE4 + dst, top);
  RD4(I4RD4 + dst, top);
  VR4(I4VR4 + dst, top);
  LD4(I4LD4 + dst, top);
  VL4(I4VL4 + dst, top);
  HD4(I4HD4 + dst, top);
  HU4(I4HU4 + dst, top);
}

//------------------------------------------------------------------------------
// Metric

#if !defined(WORK_AROUND_GCC)

#define GET_SSE_INNER(A)                                                  \
  "lw               %[temp0],    " #A "(%[a])                  \n\t"      \
  "lw               %[temp1],    " #A "(%[b])                  \n\t"      \
  "preceu.ph.qbr    %[temp2],    %[temp0]                      \n\t"      \
  "preceu.ph.qbl    %[temp0],    %[temp0]                      \n\t"      \
  "preceu.ph.qbr    %[temp3],    %[temp1]                      \n\t"      \
  "preceu.ph.qbl    %[temp1],    %[temp1]                      \n\t"      \
  "subq.ph          %[temp2],    %[temp2],    %[temp3]         \n\t"      \
  "subq.ph          %[temp0],    %[temp0],    %[temp1]         \n\t"      \
  "dpa.w.ph         $ac0,        %[temp2],    %[temp2]         \n\t"      \
  "dpa.w.ph         $ac0,        %[temp0],    %[temp0]         \n\t"

#define GET_SSE(A, B, C, D)               \
  GET_SSE_INNER(A)                        \
  GET_SSE_INNER(B)                        \
  GET_SSE_INNER(C)                        \
  GET_SSE_INNER(D)

static int SSE16x16(const uint8_t* a, const uint8_t* b) {
  int count;
  int temp0, temp1, temp2, temp3;
  __asm__ volatile (
    "mult   $zero,    $zero                            \n\t"
    GET_SSE( 0 * BPS, 4 +  0 * BPS, 8 +  0 * BPS, 12 +  0 * BPS)
    GET_SSE( 1 * BPS, 4 +  1 * BPS, 8 +  1 * BPS, 12 +  1 * BPS)
    GET_SSE( 2 * BPS, 4 +  2 * BPS, 8 +  2 * BPS, 12 +  2 * BPS)
    GET_SSE( 3 * BPS, 4 +  3 * BPS, 8 +  3 * BPS, 12 +  3 * BPS)
    GET_SSE( 4 * BPS, 4 +  4 * BPS, 8 +  4 * BPS, 12 +  4 * BPS)
    GET_SSE( 5 * BPS, 4 +  5 * BPS, 8 +  5 * BPS, 12 +  5 * BPS)
    GET_SSE( 6 * BPS, 4 +  6 * BPS, 8 +  6 * BPS, 12 +  6 * BPS)
    GET_SSE( 7 * BPS, 4 +  7 * BPS, 8 +  7 * BPS, 12 +  7 * BPS)
    GET_SSE( 8 * BPS, 4 +  8 * BPS, 8 +  8 * BPS, 12 +  8 * BPS)
    GET_SSE( 9 * BPS, 4 +  9 * BPS, 8 +  9 * BPS, 12 +  9 * BPS)
    GET_SSE(10 * BPS, 4 + 10 * BPS, 8 + 10 * BPS, 12 + 10 * BPS)
    GET_SSE(11 * BPS, 4 + 11 * BPS, 8 + 11 * BPS, 12 + 11 * BPS)
    GET_SSE(12 * BPS, 4 + 12 * BPS, 8 + 12 * BPS, 12 + 12 * BPS)
    GET_SSE(13 * BPS, 4 + 13 * BPS, 8 + 13 * BPS, 12 + 13 * BPS)
    GET_SSE(14 * BPS, 4 + 14 * BPS, 8 + 14 * BPS, 12 + 14 * BPS)
    GET_SSE(15 * BPS, 4 + 15 * BPS, 8 + 15 * BPS, 12 + 15 * BPS)
    "mflo   %[count]                                   \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [count]"=&r"(count)
    : [a]"r"(a), [b]"r"(b)
    : "memory", "hi", "lo"
  );
  return count;
}

static int SSE16x8(const uint8_t* a, const uint8_t* b) {
  int count;
  int temp0, temp1, temp2, temp3;
  __asm__ volatile (
    "mult   $zero,    $zero                            \n\t"
    GET_SSE( 0 * BPS, 4 +  0 * BPS, 8 +  0 * BPS, 12 +  0 * BPS)
    GET_SSE( 1 * BPS, 4 +  1 * BPS, 8 +  1 * BPS, 12 +  1 * BPS)
    GET_SSE( 2 * BPS, 4 +  2 * BPS, 8 +  2 * BPS, 12 +  2 * BPS)
    GET_SSE( 3 * BPS, 4 +  3 * BPS, 8 +  3 * BPS, 12 +  3 * BPS)
    GET_SSE( 4 * BPS, 4 +  4 * BPS, 8 +  4 * BPS, 12 +  4 * BPS)
    GET_SSE( 5 * BPS, 4 +  5 * BPS, 8 +  5 * BPS, 12 +  5 * BPS)
    GET_SSE( 6 * BPS, 4 +  6 * BPS, 8 +  6 * BPS, 12 +  6 * BPS)
    GET_SSE( 7 * BPS, 4 +  7 * BPS, 8 +  7 * BPS, 12 +  7 * BPS)
    "mflo   %[count]                                   \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [count]"=&r"(count)
    : [a]"r"(a), [b]"r"(b)
    : "memory", "hi", "lo"
  );
  return count;
}

static int SSE8x8(const uint8_t* a, const uint8_t* b) {
  int count;
  int temp0, temp1, temp2, temp3;
  __asm__ volatile (
    "mult   $zero,    $zero                            \n\t"
    GET_SSE(0 * BPS, 4 + 0 * BPS, 1 * BPS, 4 + 1 * BPS)
    GET_SSE(2 * BPS, 4 + 2 * BPS, 3 * BPS, 4 + 3 * BPS)
    GET_SSE(4 * BPS, 4 + 4 * BPS, 5 * BPS, 4 + 5 * BPS)
    GET_SSE(6 * BPS, 4 + 6 * BPS, 7 * BPS, 4 + 7 * BPS)
    "mflo   %[count]                                   \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [count]"=&r"(count)
    : [a]"r"(a), [b]"r"(b)
    : "memory", "hi", "lo"
  );
  return count;
}

static int SSE4x4(const uint8_t* a, const uint8_t* b) {
  int count;
  int temp0, temp1, temp2, temp3;
  __asm__ volatile (
    "mult   $zero,    $zero                            \n\t"
    GET_SSE(0 * BPS, 1 * BPS, 2 * BPS, 3 * BPS)
    "mflo   %[count]                                   \n\t"
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [count]"=&r"(count)
    : [a]"r"(a), [b]"r"(b)
    : "memory", "hi", "lo"
  );
  return count;
}

#undef GET_SSE
#undef GET_SSE_INNER

#endif  // !WORK_AROUND_GCC

#undef FILL_8_OR_16
#undef FILL_PART
#undef OUTPUT_EARLY_CLOBBER_REGS_17
#undef MUL_HALF
#undef ABS_X8
#undef ADD_SUB_HALVES_X4

//------------------------------------------------------------------------------
// Quantization
//

// macro for one pass through for loop in QuantizeBlock reading 2 values at time
// QUANTDIV macro inlined
// J - offset in bytes (kZigzag[n] * 2)
// K - offset in bytes (kZigzag[n] * 4)
// N - offset in bytes (n * 2)
// N1 - offset in bytes ((n + 1) * 2)
#define QUANTIZE_ONE(J, K, N, N1)                                         \
  "ulw         %[temp1],     " #J "(%[ppin])                 \n\t"        \
  "ulw         %[temp2],     " #J "(%[ppsharpen])            \n\t"        \
  "lhu         %[temp3],     " #K "(%[ppzthresh])            \n\t"        \
  "lhu         %[temp6],     " #K "+4(%[ppzthresh])          \n\t"        \
  "absq_s.ph   %[temp4],     %[temp1]                        \n\t"        \
  "ins         %[temp3],     %[temp6],         16,       16  \n\t"        \
  "addu.ph     %[coeff],     %[temp4],         %[temp2]      \n\t"        \
  "shra.ph     %[sign],      %[temp1],         15            \n\t"        \
  "li          %[level],     0x10001                         \n\t"        \
  "cmp.lt.ph   %[temp3],     %[coeff]                        \n\t"        \
  "lhu         %[temp1],     " #J "(%[ppiq])                 \n\t"        \
  "pick.ph     %[temp5],     %[level],         $0            \n\t"        \
  "lw          %[temp2],     " #K "(%[ppbias])               \n\t"        \
  "beqz        %[temp5],     0f                              \n\t"        \
  "lhu         %[temp3],     " #J "(%[ppq])                  \n\t"        \
  "beq         %[temp5],     %[level],         1f            \n\t"        \
  "andi        %[temp5],     %[temp5],         0x1           \n\t"        \
  "andi        %[temp4],     %[coeff],         0xffff        \n\t"        \
  "beqz        %[temp5],     2f                              \n\t"        \
  "mul         %[level],     %[temp4],         %[temp1]      \n\t"        \
  "sh          $0,           " #J "+2(%[ppin])               \n\t"        \
  "sh          $0,           " #N1 "(%[pout])                \n\t"        \
  "addu        %[level],     %[level],         %[temp2]      \n\t"        \
  "sra         %[level],     %[level],         17            \n\t"        \
  "slt         %[temp4],     %[max_level],     %[level]      \n\t"        \
  "movn        %[level],     %[max_level],     %[temp4]      \n\t"        \
  "andi        %[temp6],     %[sign],          0xffff        \n\t"        \
  "xor         %[level],     %[level],         %[temp6]      \n\t"        \
  "subu        %[level],     %[level],         %[temp6]      \n\t"        \
  "mul         %[temp5],     %[level],         %[temp3]      \n\t"        \
  "or          %[ret],       %[ret],           %[level]      \n\t"        \
  "sh          %[level],     " #N "(%[pout])                 \n\t"        \
  "sh          %[temp5],     " #J "(%[ppin])                 \n\t"        \
  "j           3f                                            \n\t"        \
"2:                                                          \n\t"        \
  "lhu         %[temp1],     " #J "+2(%[ppiq])               \n\t"        \
  "srl         %[temp5],     %[coeff],         16            \n\t"        \
  "mul         %[level],     %[temp5],         %[temp1]      \n\t"        \
  "lw          %[temp2],     " #K "+4(%[ppbias])             \n\t"        \
  "lhu         %[temp3],     " #J "+2(%[ppq])                \n\t"        \
  "addu        %[level],     %[level],         %[temp2]      \n\t"        \
  "sra         %[level],     %[level],         17            \n\t"        \
  "srl         %[temp6],     %[sign],          16            \n\t"        \
  "slt         %[temp4],     %[max_level],     %[level]      \n\t"        \
  "movn        %[level],     %[max_level],     %[temp4]      \n\t"        \
  "xor         %[level],     %[level],         %[temp6]      \n\t"        \
  "subu        %[level],     %[level],         %[temp6]      \n\t"        \
  "mul         %[temp5],     %[level],         %[temp3]      \n\t"        \
  "sh          $0,           " #J "(%[ppin])                 \n\t"        \
  "sh          $0,           " #N "(%[pout])                 \n\t"        \
  "or          %[ret],       %[ret],           %[level]      \n\t"        \
  "sh          %[temp5],     " #J "+2(%[ppin])               \n\t"        \
  "sh          %[level],     " #N1 "(%[pout])                \n\t"        \
  "j           3f                                            \n\t"        \
"1:                                                          \n\t"        \
  "lhu         %[temp1],     " #J "(%[ppiq])                 \n\t"        \
  "lw          %[temp2],     " #K "(%[ppbias])               \n\t"        \
  "ulw         %[temp3],     " #J "(%[ppq])                  \n\t"        \
  "andi        %[temp5],     %[coeff],         0xffff        \n\t"        \
  "srl         %[temp0],     %[coeff],         16            \n\t"        \
  "lhu         %[temp6],     " #J "+2(%[ppiq])               \n\t"        \
  "lw          %[coeff],     " #K "+4(%[ppbias])             \n\t"        \
  "mul         %[level],     %[temp5],         %[temp1]      \n\t"        \
  "mul         %[temp4],     %[temp0],         %[temp6]      \n\t"        \
  "addu        %[level],     %[level],         %[temp2]      \n\t"        \
  "addu        %[temp4],     %[temp4],         %[coeff]      \n\t"        \
  "precrq.ph.w %[level],     %[temp4],         %[level]      \n\t"        \
  "shra.ph     %[level],     %[level],         1             \n\t"        \
  "cmp.lt.ph   %[max_level1],%[level]                        \n\t"        \
  "pick.ph     %[level],     %[max_level],     %[level]      \n\t"        \
  "xor         %[level],     %[level],         %[sign]       \n\t"        \
  "subu.ph     %[level],     %[level],         %[sign]       \n\t"        \
  "mul.ph      %[temp3],     %[level],         %[temp3]      \n\t"        \
  "or          %[ret],       %[ret],           %[level]      \n\t"        \
  "sh          %[level],     " #N "(%[pout])                 \n\t"        \
  "srl         %[level],     %[level],         16            \n\t"        \
  "sh          %[level],     " #N1 "(%[pout])                \n\t"        \
  "usw         %[temp3],     " #J "(%[ppin])                 \n\t"        \
  "j           3f                                            \n\t"        \
"0:                                                          \n\t"        \
  "sh          $0,           " #N "(%[pout])                 \n\t"        \
  "sh          $0,           " #N1 "(%[pout])                \n\t"        \
  "usw         $0,           " #J "(%[ppin])                 \n\t"        \
"3:                                                          \n\t"

static int QuantizeBlock(int16_t in[16], int16_t out[16],
                         const VP8Matrix* const mtx) {
  int temp0, temp1, temp2, temp3, temp4, temp5,temp6;
  int sign, coeff, level;
  int max_level = MAX_LEVEL;
  int max_level1 = max_level << 16 | max_level;
  int ret = 0;

  int16_t* ppin             = &in[0];
  int16_t* pout             = &out[0];
  const uint16_t* ppsharpen = &mtx->sharpen_[0];
  const uint32_t* ppzthresh = &mtx->zthresh_[0];
  const uint16_t* ppq       = &mtx->q_[0];
  const uint16_t* ppiq      = &mtx->iq_[0];
  const uint32_t* ppbias    = &mtx->bias_[0];

  __asm__ volatile (
    QUANTIZE_ONE( 0,  0,  0,  2)
    QUANTIZE_ONE( 4,  8, 10, 12)
    QUANTIZE_ONE( 8, 16,  4,  8)
    QUANTIZE_ONE(12, 24, 14, 24)
    QUANTIZE_ONE(16, 32,  6, 16)
    QUANTIZE_ONE(20, 40, 22, 26)
    QUANTIZE_ONE(24, 48, 18, 20)
    QUANTIZE_ONE(28, 56, 28, 30)

    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1),
      [temp2]"=&r"(temp2), [temp3]"=&r"(temp3),
      [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [sign]"=&r"(sign), [coeff]"=&r"(coeff),
      [level]"=&r"(level), [temp6]"=&r"(temp6), [ret]"+&r"(ret)
    : [ppin]"r"(ppin), [pout]"r"(pout), [max_level1]"r"(max_level1),
      [ppiq]"r"(ppiq), [max_level]"r"(max_level),
      [ppbias]"r"(ppbias), [ppzthresh]"r"(ppzthresh),
      [ppsharpen]"r"(ppsharpen), [ppq]"r"(ppq)
    : "memory", "hi", "lo"
  );

  return (ret != 0);
}

static int Quantize2Blocks(int16_t in[32], int16_t out[32],
                           const VP8Matrix* const mtx) {
  int nz;
  nz  = QuantizeBlock(in + 0 * 16, out + 0 * 16, mtx) << 0;
  nz |= QuantizeBlock(in + 1 * 16, out + 1 * 16, mtx) << 1;
  return nz;
}

#undef QUANTIZE_ONE

// macro for one horizontal pass in FTransformWHT
// temp0..temp7 holds tmp[0]..tmp[15]
// A, B, C, D - offset in bytes to load from in buffer
// TEMP0, TEMP1 - registers for corresponding tmp elements
#define HORIZONTAL_PASS_WHT(A, B, C, D, TEMP0, TEMP1)                          \
  "lh              %[" #TEMP0 "],  " #A "(%[in])            \n\t"              \
  "lh              %[" #TEMP1 "],  " #B "(%[in])            \n\t"              \
  "lh              %[temp8],     " #C "(%[in])              \n\t"              \
  "lh              %[temp9],     " #D "(%[in])              \n\t"              \
  "ins             %[" #TEMP1 "],  %[" #TEMP0 "],  16,  16  \n\t"              \
  "ins             %[temp9],     %[temp8],     16,  16      \n\t"              \
  "subq.ph         %[temp8],     %[" #TEMP1 "],  %[temp9]   \n\t"              \
  "addq.ph         %[temp9],     %[" #TEMP1 "],  %[temp9]   \n\t"              \
  "precrq.ph.w     %[" #TEMP0 "],  %[temp8],     %[temp9]   \n\t"              \
  "append          %[temp8],     %[temp9],     16           \n\t"              \
  "subq.ph         %[" #TEMP1 "],  %[" #TEMP0 "],  %[temp8] \n\t"              \
  "addq.ph         %[" #TEMP0 "],  %[" #TEMP0 "],  %[temp8] \n\t"              \
  "rotr            %[" #TEMP1 "],  %[" #TEMP1 "],  16       \n\t"

// macro for one vertical pass in FTransformWHT
// temp0..temp7 holds tmp[0]..tmp[15]
// A, B, C, D - offsets in bytes to store to out buffer
// TEMP0, TEMP2, TEMP4 and TEMP6 - registers for corresponding tmp elements
#define VERTICAL_PASS_WHT(A, B, C, D, TEMP0, TEMP2, TEMP4, TEMP6)              \
  "addq.ph         %[temp8],     %[" #TEMP0 "],  %[" #TEMP4 "]    \n\t"        \
  "addq.ph         %[temp9],     %[" #TEMP2 "],  %[" #TEMP6 "]    \n\t"        \
  "subq.ph         %[" #TEMP2 "],  %[" #TEMP2 "],  %[" #TEMP6 "]  \n\t"        \
  "subq.ph         %[" #TEMP6 "],  %[" #TEMP0 "],  %[" #TEMP4 "]  \n\t"        \
  "addqh.ph        %[" #TEMP0 "],  %[temp8],     %[temp9]         \n\t"        \
  "subqh.ph        %[" #TEMP4 "],  %[" #TEMP6 "],  %[" #TEMP2 "]  \n\t"        \
  "addqh.ph        %[" #TEMP2 "],  %[" #TEMP2 "],  %[" #TEMP6 "]  \n\t"        \
  "subqh.ph        %[" #TEMP6 "],  %[temp8],     %[temp9]         \n\t"        \
  "usw             %[" #TEMP0 "],  " #A "(%[out])                 \n\t"        \
  "usw             %[" #TEMP2 "],  " #B "(%[out])                 \n\t"        \
  "usw             %[" #TEMP4 "],  " #C "(%[out])                 \n\t"        \
  "usw             %[" #TEMP6 "],  " #D "(%[out])                 \n\t"

static void FTransformWHT(const int16_t* in, int16_t* out) {
  int temp0, temp1, temp2, temp3, temp4;
  int temp5, temp6, temp7, temp8, temp9;

  __asm__ volatile (
    HORIZONTAL_PASS_WHT(  0,  32,  64,  96, temp0, temp1)
    HORIZONTAL_PASS_WHT(128, 160, 192, 224, temp2, temp3)
    HORIZONTAL_PASS_WHT(256, 288, 320, 352, temp4, temp5)
    HORIZONTAL_PASS_WHT(384, 416, 448, 480, temp6, temp7)
    VERTICAL_PASS_WHT(0,  8, 16, 24, temp0, temp2, temp4, temp6)
    VERTICAL_PASS_WHT(4, 12, 20, 28, temp1, temp3, temp5, temp7)
    : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
      [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
      [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8),
      [temp9]"=&r"(temp9)
    : [in]"r"(in), [out]"r"(out)
    : "memory"
  );
}

#undef VERTICAL_PASS_WHT
#undef HORIZONTAL_PASS_WHT

// macro for converting coefficients to bin
// convert 8 coeffs at time
// A, B, C, D - offsets in bytes to load from out buffer
#define CONVERT_COEFFS_TO_BIN(A, B, C, D)                                      \
  "ulw        %[temp0],  " #A "(%[out])                \n\t"                   \
  "ulw        %[temp1],  " #B "(%[out])                \n\t"                   \
  "ulw        %[temp2],  " #C "(%[out])                \n\t"                   \
  "ulw        %[temp3],  " #D "(%[out])                \n\t"                   \
  "absq_s.ph  %[temp0],  %[temp0]                      \n\t"                   \
  "absq_s.ph  %[temp1],  %[temp1]                      \n\t"                   \
  "absq_s.ph  %[temp2],  %[temp2]                      \n\t"                   \
  "absq_s.ph  %[temp3],  %[temp3]                      \n\t"                   \
  /* TODO(skal): add rounding ? shra_r.ph : shra.ph */                         \
  /*             for following 4 instructions       */                         \
  "shra.ph    %[temp0],  %[temp0],    3                \n\t"                   \
  "shra.ph    %[temp1],  %[temp1],    3                \n\t"                   \
  "shra.ph    %[temp2],  %[temp2],    3                \n\t"                   \
  "shra.ph    %[temp3],  %[temp3],    3                \n\t"                   \
  "shll_s.ph  %[temp0],  %[temp0],    10               \n\t"                   \
  "shll_s.ph  %[temp1],  %[temp1],    10               \n\t"                   \
  "shll_s.ph  %[temp2],  %[temp2],    10               \n\t"                   \
  "shll_s.ph  %[temp3],  %[temp3],    10               \n\t"                   \
  "shrl.ph    %[temp0],  %[temp0],    10               \n\t"                   \
  "shrl.ph    %[temp1],  %[temp1],    10               \n\t"                   \
  "shrl.ph    %[temp2],  %[temp2],    10               \n\t"                   \
  "shrl.ph    %[temp3],  %[temp3],    10               \n\t"                   \
  "shll.ph    %[temp0],  %[temp0],    2                \n\t"                   \
  "shll.ph    %[temp1],  %[temp1],    2                \n\t"                   \
  "shll.ph    %[temp2],  %[temp2],    2                \n\t"                   \
  "shll.ph    %[temp3],  %[temp3],    2                \n\t"                   \
  "ext        %[temp4],  %[temp0],    0,       16      \n\t"                   \
  "ext        %[temp0],  %[temp0],    16,      16      \n\t"                   \
  "addu       %[temp4],  %[temp4],    %[dist]          \n\t"                   \
  "addu       %[temp0],  %[temp0],    %[dist]          \n\t"                   \
  "ext        %[temp5],  %[temp1],    0,       16      \n\t"                   \
  "lw         %[temp8],  0(%[temp4])                   \n\t"                   \
  "ext        %[temp1],  %[temp1],    16,      16      \n\t"                   \
  "addu       %[temp5],  %[temp5],    %[dist]          \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp4])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp0])                   \n\t"                   \
  "addu       %[temp1],  %[temp1],    %[dist]          \n\t"                   \
  "ext        %[temp6],  %[temp2],    0,       16      \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp0])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp5])                   \n\t"                   \
  "ext        %[temp2],  %[temp2],    16,      16      \n\t"                   \
  "addu       %[temp6],  %[temp6],    %[dist]          \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp5])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp1])                   \n\t"                   \
  "addu       %[temp2],  %[temp2],    %[dist]          \n\t"                   \
  "ext        %[temp7],  %[temp3],    0,       16      \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp1])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp6])                   \n\t"                   \
  "ext        %[temp3],  %[temp3],    16,      16      \n\t"                   \
  "addu       %[temp7],  %[temp7],    %[dist]          \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp6])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp2])                   \n\t"                   \
  "addu       %[temp3],  %[temp3],    %[dist]          \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp2])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp7])                   \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp7])                   \n\t"                   \
  "lw         %[temp8],  0(%[temp3])                   \n\t"                   \
  "addiu      %[temp8],  %[temp8],    1                \n\t"                   \
  "sw         %[temp8],  0(%[temp3])                   \n\t"

static void CollectHistogram(const uint8_t* ref, const uint8_t* pred,
                             int start_block, int end_block,
                             VP8Histogram* const histo) {
  int j;
  int distribution[MAX_COEFF_THRESH + 1] = { 0 };
  const int max_coeff = (MAX_COEFF_THRESH << 16) + MAX_COEFF_THRESH;
  for (j = start_block; j < end_block; ++j) {
    int16_t out[16];
    int temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7, temp8;

    VP8FTransform(ref + VP8DspScan[j], pred + VP8DspScan[j], out);

    // Convert coefficients to bin.
    __asm__ volatile (
      CONVERT_COEFFS_TO_BIN( 0,  4,  8, 12)
      CONVERT_COEFFS_TO_BIN(16, 20, 24, 28)
      : [temp0]"=&r"(temp0), [temp1]"=&r"(temp1), [temp2]"=&r"(temp2),
        [temp3]"=&r"(temp3), [temp4]"=&r"(temp4), [temp5]"=&r"(temp5),
        [temp6]"=&r"(temp6), [temp7]"=&r"(temp7), [temp8]"=&r"(temp8)
      : [dist]"r"(distribution), [out]"r"(out), [max_coeff]"r"(max_coeff)
      : "memory"
    );
  }
  VP8SetHistogramData(distribution, histo);
}

#undef CONVERT_COEFFS_TO_BIN

//------------------------------------------------------------------------------
// Entry point

extern void VP8EncDspInitMIPSdspR2(void);

WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspInitMIPSdspR2(void) {
  VP8FTransform = FTransform;
  VP8ITransform = ITransform;
  VP8TDisto4x4 = Disto4x4;
  VP8TDisto16x16 = Disto16x16;
  VP8EncPredLuma16 = Intra16Preds;
  VP8EncPredChroma8 = IntraChromaPreds;
  VP8EncPredLuma4 = Intra4Preds;
#if !defined(WORK_AROUND_GCC)
  VP8SSE16x16 = SSE16x16;
  VP8SSE8x8 = SSE8x8;
  VP8SSE16x8 = SSE16x8;
  VP8SSE4x4 = SSE4x4;
#endif
  VP8EncQuantizeBlock = QuantizeBlock;
  VP8EncQuantize2Blocks = Quantize2Blocks;
  VP8FTransformWHT = FTransformWHT;
  VP8CollectHistogram = CollectHistogram;
}

#else  // !WEBP_USE_MIPS_DSP_R2

WEBP_DSP_INIT_STUB(VP8EncDspInitMIPSdspR2)

#endif  // WEBP_USE_MIPS_DSP_R2