Android - notifyDataSetChange from within custom ArrayAdapter class

I have a custom ArrayAdapter called InteractiveArrayAdapter that adds buttons and button listeners to each item in a list view. Inside the adapter there is a getView method that creates a view inflator. Inside of here is where my button is created and the buttonListener is created. When the button is clicked i delete the element in my ArrayList associated with that button. The problem is that i cant figure out how to call notifyDataSetChange from within this OnClick method, or another way to notify the adapter that the listView needs to be updated.

Custom adapter:

public class InteractiveArrayAdapter extends ArrayAdapter<String> {

  private final List<String> list;
  private final Activity context;
  private ListView listV;

  public InteractiveArrayAdapter(Activity context, List<String> list) {
    super(context, R.layout.rowbuttonlayout, list);
    this.context = context;
    this.list = list;

  }

  static class ViewHolder {
    protected TextView text;
    protected Button button;
  }

  @Override
  public View getView(int position, View convertView, ViewGroup parent) {
    View view = null;

    if (convertView == null) {
      LayoutInflater inflator = context.getLayoutInflater();
      view = inflator.inflate(R.layout.rowbuttonlayout, null);
      final ViewHolder viewHolder = new ViewHolder();
      viewHolder.text = (TextView) view.findViewById(R.id.label);
      viewHolder.button = (Button) view.findViewById(R.id.add);
      viewHolder.button
          .setOnClickListener(new Button.OnClickListener() {

//            @Override
//            public void onButtonClicked(Button buttonView,
//                boolean isChecked) {
//              String element = (String) viewHolder.button
//                  .getTag();
//              element.setSelected(buttonView.isChecked());
//
//            }

            public void onClick(View v) {
                String tag = (String) viewHolder.button.getTag();
                list.remove(list.indexOf(tag));
                // TODO Auto-generated method stub

            }
          });
      view.setTag(viewHolder);
      viewHolder.button.setTag(list.get(position));
    } else {
      view = convertView;
      ((ViewHolder) view.getTag()).button.setTag(list.get(position));
    }
    ViewHolder holder = (ViewHolder) view.getTag();
    holder.text.setText(list.get(position));

    return view;
  }
} 

Activity:

public class InventoryActivity extends Activity {
    public Inventory appInv;
    private ListView lv1 = null;
    private ListView lv2 = null;

    /** Called when the activity is first created. */
    @Override
    public void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        SharedPreferences prefs = getPreferences(MODE_PRIVATE);
        appInv = new Inventory(prefs);

        setContentView(R.layout.inventory);

        // Get UI references.
        //
        lv1 = (ListView) findViewById (R.id.list1);
        lv2 = (ListView) findViewById (R.id.list2);
        ArrayAdapter<String> adapter1 = new InteractiveArrayAdapter (this, appInv.inventory);
        ArrayAdapter<String> adapter2 = new InteractiveArrayAdapter (this,  appInv.ingredients);
        lv1.setAdapter(adapter1);
        lv2.setAdapter(adapter2);
        // TODO Auto-generated method stub
    }
}