ALSA: firewire-lib: fix buffer-over-run when detecting packet discontinuity

When detecting packet discontinuity, handle_in_packet() returns minus value
and this value is assigned to unsigned int variable, then the variable has
huge value. As a result, the variable causes buffer-over-run in
handle_out_packet(). This brings invalid page request and system hangup.

This commit fixes the bug to add a new argument into handle_in_packet()
and the number of handled data blocks is assignd to it. The function
return value is just used to check error.

I also considered to change the type of local variable to 'int' in
in_stream_callback(). This idea is based on type-conversion in C standard,
while it may cause future problems when adding more works. Thus, I dropped
this idea.

Fixes: 6fc6b9ce41c6('ALSA: firewire-lib: pass the number of data blocks in incoming packets to outgoing packets')
Reported-by: Dan Carpenter <[e-mail]>
Signed-off-by: Takashi Sakamoto <[e-mail]>
Signed-off-by: Takashi Iwai <[e-mail]>

sound/firewire/amdtp.c

···
 }
 
 static int handle_in_packet(struct amdtp_stream *s,
-			    unsigned int payload_quadlets, __be32 *buffer)
+			    unsigned int payload_quadlets, __be32 *buffer,
+			    unsigned int *data_blocks)
 {
 	u32 cip_header[2];
-	unsigned int data_blocks;
 	unsigned int data_block_quadlets, data_block_counter, dbc_interval;
 	struct snd_pcm_substream *pcm = NULL;
 	bool lost;
···
 		dev_info_ratelimited(&s->unit->device,
 				"Invalid CIP header for AMDTP: %08X:%08X\n",
 				cip_header[0], cip_header[1]);
-		data_blocks = 0;
+		*data_blocks = 0;
 		goto end;
 	}
 
···
 	if (payload_quadlets < 3 ||
 	    ((cip_header[1] & CIP_FDF_MASK) ==
 				(AMDTP_FDF_NO_DATA << CIP_FDF_SHIFT))) {
-		data_blocks = 0;
+		*data_blocks = 0;
 	} else {
 		data_block_quadlets =
 			(cip_header[0] & CIP_DBS_MASK) >> CIP_DBS_SHIFT;
···
 		if (s->flags & CIP_WRONG_DBS)
 			data_block_quadlets = s->data_block_quadlets;
 
-		data_blocks = (payload_quadlets - 2) / data_block_quadlets;
+		*data_blocks = (payload_quadlets - 2) / data_block_quadlets;
 	}
 
 	/* Check data block counter continuity */
 	data_block_counter = cip_header[0] & CIP_DBC_MASK;
-	if (data_blocks == 0 && (s->flags & CIP_EMPTY_HAS_WRONG_DBC) &&
+	if (*data_blocks == 0 && (s->flags & CIP_EMPTY_HAS_WRONG_DBC) &&
 	    s->data_block_counter != UINT_MAX)
 		data_block_counter = s->data_block_counter;
 
···
 	} else if (!(s->flags & CIP_DBC_IS_END_EVENT)) {
 		lost = data_block_counter != s->data_block_counter;
 	} else {
-		if ((data_blocks > 0) && (s->tx_dbc_interval > 0))
+		if ((*data_blocks > 0) && (s->tx_dbc_interval > 0))
 			dbc_interval = s->tx_dbc_interval;
 		else
-			dbc_interval = data_blocks;
+			dbc_interval = *data_blocks;
 
 		lost = data_block_counter !=
 		       ((s->data_block_counter + dbc_interval) & 0xff);
···
 		return -EIO;
 	}
 
-	if (data_blocks > 0) {
+	if (*data_blocks > 0) {
 		buffer += 2;
 
 		pcm = ACCESS_ONCE(s->pcm);
 		if (pcm)
-			s->transfer_samples(s, pcm, buffer, data_blocks);
+			s->transfer_samples(s, pcm, buffer, *data_blocks);
 
 		if (s->midi_ports)
-			read_midi_messages(s, buffer, data_blocks);
+			read_midi_messages(s, buffer, *data_blocks);
 	}
 
 	if (s->flags & CIP_DBC_IS_END_EVENT)
 		s->data_block_counter = data_block_counter;
 	else
 		s->data_block_counter =
-				(data_block_counter + data_blocks) & 0xff;
+				(data_block_counter + *data_blocks) & 0xff;
 end:
 	if (queue_in_packet(s) < 0)
 		return -EIO;
 
 	if (pcm)
-		update_pcm_pointers(s, pcm, data_blocks);
+		update_pcm_pointers(s, pcm, *data_blocks);
 
-	return data_blocks;
+	return 0;
 }
 
 static void out_stream_callback(struct fw_iso_context *context, u32 cycle,
···
 			break;
 		}
 
-		data_blocks = handle_in_packet(s, payload_quadlets, buffer);
-		if (data_blocks < 0) {
+		if (handle_in_packet(s, payload_quadlets, buffer,
+							&data_blocks) < 0) {
 			s->packet_index = -1;
 			break;
 		}