Tracking objects using histogram data in OpenCV

I am trying to track objects inside an image using histogram data from the object. I pass in a reference image to get the histogram data and store it in a Mat. From there I load in an image and try and use the histogram data to detect the object. The problem I am coming with is not only is it not tracking the object, but it's not updating the detection. If I load image "1.jpg" the detection will claim that the object is in the top right corner when it's in the bottom left. When I pass in the second image the detection field does not move at all. This continues for the next batch of images as well. Below is a code snippet of my application.

This is being done in a Windows 7 32-bit environment using OpenCV2.3 in VS2010. Thanks in advance for any help

int main( int argc, char** argv )
{
    vector<string> szFileNames;
    IplImage* Image;
    Mat img, hist, backproj;
    Rect trackWindow;

    // Load histogram data
    hist = ImageHistogram("C:/Users/seb/Documents/redbox1.jpg", backproj);

    Image = cvLoadImage("C:/Users/seb/Documents/1.jpg");
        img = Mat(Image);
    trackWindow = Rect(0, 0, Image->width, Image->height);

    imshow("Histogram", hist);

    while(true)
    {
        Detection(img, backproj, trackWindow);
        imshow("Image", img);

        char c = cvWaitKey(1);

        switch(c)
        {
            case 32:
            {
                cvReleaseImage(&Image);
                Image = cvLoadImage("C:/Users/seb/Documents/redbox2.jpg");
                img = Mat(Image);
                break;
            }
        }
    }

    cvReleaseImage(&Image);

    // Destroy all windows
    cvDestroyWindow("Histogram");
    cvDestroyWindow("Image");

    return 0;
}

Mat ImageHistogram(string szFilename, Mat& backproj)
{
    // Create histogram values
    int vmin = 10;
    int vmax = 256;
    int smin = 30;
    int hsize = 16;
    float hranges[] = {0,180};
    const float* phranges = hranges;

    // Load the image
    IplImage* Image = cvLoadImage(szFilename.c_str());
    Rect rect = Rect(0, 0, Image->width, Image->height);

    // Convert Image to a matrix
    Mat ImageMat = Mat(Image);

    // Create and initialize the Histogram
    Mat hsv, mask, hue, hist, histimg = Mat::zeros(200, 320, CV_8UC3);
    cvtColor(ImageMat, hsv, CV_BGR2HSV);

    // Create and adjust the histogram values
    inRange(hsv, Scalar(0, smin, vmin), Scalar(180, 256, vmax), mask);
    int ch[] = {0, 0};

    hue.create(hsv.size(), hsv.depth());
    mixChannels(&hsv, 1, &hue, 1, ch, 1);

    Mat roi(hue, rect), maskroi(mask, rect);
    calcHist(&roi, 1, 0, maskroi, hist, 1, &hsize, &phranges);
    normalize(hist, hist, 0, 255, CV_MINMAX);

    histimg = Scalar::all(0);
    int binW = histimg.cols / hsize;
    Mat buf(1, hsize, CV_8UC3);
    for( int i = 0; i < hsize; i++ )
        buf.at<Vec3b>(i) = Vec3b(saturate_cast<uchar>(i*180./hsize), 255, 255);
    cvtColor(buf, buf, CV_HSV2BGR);

    for( int i = 0; i < hsize; i++ )
    {
        int val = saturate_cast<int>(hist.at<float>(i)*histimg.rows/255);
        rectangle( histimg, Point(i*binW,histimg.rows),
            Point((i+1)*binW,histimg.rows - val),
            Scalar(buf.at<Vec3b>(i)), -1, 8 );
    }

    calcBackProject(&hue, 1, 0, hist, backproj, &phranges);
    backproj &= mask;

    cvReleaseImage(&Image);

    return histimg;
}

void Detection(Mat& image, Mat& backproj, Rect& trackWindow)
{
    RotatedRect trackBox = CamShift(backproj, trackWindow, TermCriteria( CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1 ));
    int test2 = trackWindow.area();
    if(trackBox.size.height > 0 && trackBox.size.width > 0)
    {
        if( trackWindow.area() <= 1 )
        {
            int cols = backproj.cols, rows = backproj.rows, r = (MIN(cols, rows) + 5)/6;
            trackWindow = Rect(trackWindow.x - r, trackWindow.y - r,
                trackWindow.x + r, trackWindow.y + r) &
                Rect(0, 0, cols, rows);
        }

        int test = trackBox.size.area();
        if(test >= 1)
        {
            rectangle(image, trackBox.boundingRect(), Scalar(255,0,0), 3, CV_AA);
            ellipse( image, trackBox, Scalar(0,0,255), 3, CV_AA );
        }
    }
}