media: vicodec: improve handling of uncompressable planes

Exit the loop immediately once it is clear that the plane
cannot be compressed. Also clear the PCODED bit and fix the
PCODED check (it should check for the bit) in the caller code.

Signed-off-by: Hans Verkuil <[e-mail]>
Signed-off-by: Mauro Carvalho Chehab <[e-mail]>

drivers/media/platform/vicodec/vicodec-codec.c

···
 			input += 8 * input_step;
 			refp += 8 * 8;
 
-			if (encoding & FRAME_UNENCODED)
-				continue;
-
 			size = rlc(cf->coeffs, *rlco, blocktype);
 			if (last_size == size &&
 			    !memcmp(*rlco + 1, *rlco - size + 1, 2 * size - 2)) {
···
 			} else {
 				*rlco += size;
 			}
-			if (*rlco >= rlco_max)
+			if (*rlco >= rlco_max) {
 				encoding |= FRAME_UNENCODED;
+				goto exit_loop;
+			}
 			last_size = size;
 		}
 		input += width * 7 * input_step;
 	}
+
+exit_loop:
 	if (encoding & FRAME_UNENCODED) {
 		u8 *out = (u8 *)rlco_start;
 
···
 		for (i = 0; i < height * width; i++, input += input_step)
 			*out++ = (*input == 0xff) ? 0xfe : *input;
 		*rlco = (__be16 *)out;
+		encoding &= ~FRAME_PCODED;
 	}
 	return encoding;
 }

drivers/media/platform/vicodec/vicodec-core.c

···
 
 	encoding = encode_frame(&rf, &ctx->ref_frame, &cf, !ctx->gop_cnt,
 				ctx->gop_cnt == ctx->gop_size - 1);
-	if (encoding != FRAME_PCODED)
+	if (!(encoding & FRAME_PCODED))
 		ctx->gop_cnt = 0;
 	if (++ctx->gop_cnt >= ctx->gop_size)
 		ctx->gop_cnt = 0;