Deleting element from a listView removes last element instead of target element

The program is suppose to add and delete views from a list view. Objects (views) are added with an add button and deleted with a long hold pop up window. Everything works but when I delete an object it deletes from the end of the arraylist and not the object I'm long holding on.

I'm aware you can tag or index objects but I don't know how to implement either option from how I designed the program.

MainActivity.java

public class MainActivity extends AppCompatActivity {
    // Construct the data source
    ArrayList<User> arrayOfUsers = new ArrayList<User>();
    
    ListView listview;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);

        // Create the adapter to convert the array to views
        UsersAdapter adapter = new UsersAdapter(this, arrayOfUsers);
        // Attach the adapter to a ListView
        ListView listView = (ListView) findViewById(R.id.book_list);
        listView.setAdapter(adapter);

        Button add = (Button)findViewById(R.id.add);
        // Activates method on yes button click
        add.setOnClickListener(new View.OnClickListener(){
            @Override
            public void onClick(View view){
                arrayOfUsers.add(new User("test", "test"));
                adapter.notifyDataSetChanged();
            }
        });

        listView.setOnItemLongClickListener(new AdapterView.OnItemLongClickListener() {

            @Override
            public boolean onItemLongClick(AdapterView<?> adapterView, View view, int i, long l) {
                final int itemToDelete = i;
                // To delete the data from the App
                new AlertDialog.Builder(MainActivity.this)
                        .setIcon(android.R.drawable.ic_dialog_alert)
                        .setTitle("Are you sure?")
                        .setMessage("Do you want to delete this note?")
                        .setPositiveButton("Yes", new DialogInterface.OnClickListener() {

                            @Override
                            public void onClick(DialogInterface dialogInterface, int i) {
                                arrayOfUsers.remove(itemToDelete);
                                adapter.notifyDataSetChanged();
                            }
                        }).setNegativeButton("No", null).show();
                return true;
            }
        });
    }
}

UsersAdapter.java

public class UsersAdapter extends ArrayAdapter<User> {
    private ArrayList<User> users;
    public UsersAdapter(Context context, ArrayList<User> users) {
        super(context, 0, users);
        this.users = users;
    }

    @Override
    public View getView(int position, View convertView, ViewGroup parent) {
        // Get the data item for this position
        User user = getItem(position);
        // Check if an existing view is being reused, otherwise inflate the view
        if (convertView == null) {
            convertView = LayoutInflater.from(getContext()).inflate(R.layout.list_item, parent, false);
        }
        // Lookup view for data population
        TextView tvName = (TextView) convertView.findViewById(R.id.tvName);
        TextView tvHome = (TextView) convertView.findViewById(R.id.tvHome);
        // Populate the data into the template view using the data object
        tvName.setText(user.name);
        tvHome.setText(user.hometown);

        EditText input_weight = (EditText) convertView.findViewById(R.id.weight);

        Button save = (Button) convertView.findViewById(R.id.save);
        // Activates method on yes button click
        save.setOnClickListener(new View.OnClickListener(){
            @Override
            public void onClick(View view){
                SharedPreferences pref = PreferenceManager.getDefaultSharedPreferences(getContext().getApplicationContext());
                SharedPreferences.Editor editor = pref.edit();
                String bw = String.valueOf(input_weight.getText());
                editor.putString("input_weight", bw);
                editor.commit(); // Saves body weight
                bw = pref.getString("body_weight", bw); // Gets body weight
                tvHome.setText(bw); // Displays body weight
            }
        });

        // Return the completed view to render on screen
        return convertView;
    }
}

activity_main.xml

<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".MainActivity">

    <ListView
        android:id="@+id/book_list"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:layout_marginTop="0dp"
        app:layout_constraintStart_toStartOf="parent"
        app:layout_constraintTop_toTopOf="parent"
        tools:listitem="@layout/list_item"/>

    <Button
        android:id="@+id/add"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:text="ADD"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toEndOf="parent"
        app:layout_constraintHorizontal_bias="1.0"
        app:layout_constraintStart_toStartOf="parent"
        app:layout_constraintTop_toTopOf="parent" />

</androidx.constraintlayout.widget.ConstraintLayout>

list_item.xml

<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    xmlns:tools="http://schemas.android.com/tools"
    tools:context=".MainActivity">
    <TextView
        android:id="@+id/tvName"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:text="Name" />
    <EditText
        android:id="@+id/weight"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:ems="10"
        android:inputType="textPersonName"
        android:hint="Weight" />
    <TextView
        android:id="@+id/tvHome"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:text="HomeTown" />
    <Button
        android:id="@+id/save"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:text="Save"
        android:focusable="false"
        />

</LinearLayout>