CAM detach races with USB memory sticks

[Hans Petter Selasky]

Hi,

I've started looking into some attach/detach stress tests for USB mass storage devices.
During my bughunting I've found the following issue:

In /sys/cam/cam_xpt.c:

4569 	  	  	void
4570 	  	  	xpt_release_device(struct cam_ed *device)
4571 	  	  	{
4572 	  	  	
4573 	mjacob 	224806 	if (device->refcount == 1) {
4574 	gibbs 	39212 	struct cam_devq *devq;
4575 	  	  	
4576 	gibbs 	44500 	if (device->alloc_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX
4577 	  	  	|| device->send_ccb_entry.pinfo.index != CAM_UNQUEUED_INDEX)
4578 	  	  	panic("Removing device while still queued for ccbs");
4579 	gibbs 	49927 	
4580 	  	  	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)

XXXX HERE XXXX

4581 	mjacob 	224806 	callout_stop(&device->callout);


4583 	mav 	198748 	TAILQ_REMOVE(&device->target->ed_entries, device,links);
4584 	  	  	device->target->generation++;
4585 	  	  	device->target->bus->sim->max_ccbs -= device->ccbq.devq_openings;
4586 	trasz 	186184 	/* Release our slot in the devq */
4587 	mav 	198748 	devq = device->target->bus->sim->devq;
4588 	trasz 	186184 	cam_devq_resize(devq, devq->alloc_queue.array_size - 1);
4589 	avatar 	147571 	camq_fini(&device->drvq);
4590 	mav 	198377 	cam_ccbq_fini(&device->ccbq);
4591 	ken 	230590 	/*
4592 	  	  	* Free allocated memory. free(9) does nothing if the
4593 	  	  	* supplied pointer is NULL, so it is safe to call without
4594 	  	  	* checking.
4595 	  	  	*/
4596 	  	  	free(device->supported_vpds, M_CAMXPT);
4597 	  	  	free(device->device_id, M_CAMXPT);
4598 	  	  	free(device->physpath, M_CAMXPT);
4599 	  	  	free(device->rcap_buf, M_CAMXPT);
4600 	  	  	free(device->serial_num, M_CAMXPT);
4601 	  	  	
4602 	mav 	198748 	xpt_release_target(device->target);
4603 	avatar 	147723 	free(device, M_CAMXPT);


Remember our OS is SMP now, so callout_drain() must be used on all callouts before 
the structure where they are located are freed. Above I only see callout_stop(). The
problem is that xpt_release_device() appears to be called locked and callout_drain()
requires unlocked context! How do we solve this? I don't know so much about the CAM
layer and I hope that someone who does can give a hand here.

--HPS