How can i rotate an object based on the finger position?

I am using Lens Studio to build an AR effect and trying to use the position of the hand tracker fingertip to make a cube rotate. For example, as I move my finger on the x-axis the cube rotates in the y-axis in the direction of the finger.

I tried the following code but there is probably something I'm missing because it only works correctly on one face of the cube.

var object = script.getSceneObject();
var transform = object.getTransform();
var rotation = transform.getWorldRotation();

var isPinch = false;

function update() {
  var hand = global.handTracking.getHand();

  if (hand === undefined) {
    return;
  }

  detectPinch(hand);

  if (isPinch) {
    rotate(hand);
  }
}

function detectPinch(hand) {
  var thumb = hand.thumb.tip.position;
  var index = hand.indexFinger.tip.position;
  var distance = thumb.distance(index);
  var threshold = 3;

  if (distance < threshold) {
    isPinch = true;
  } else {
    isPinch = false;
  }
}

function rotate(hand) {
  var pos = hand.indexFinger.tip.position;

  var q1 = quat.angleAxis(degToRad(pos.y) * rotSens, vec3.left());
  var q2 = quat.angleAxis(degToRad(pos.x) * rotSens, vec3.up());
  var q3 = q1.multiply(q2);

  transform.setWorldRotation(q3.multiply(rotation));
}

function degToRad(degrees) {
  var pi = Math.PI;
  return degrees * (pi / 180);
}

script.createEvent("UpdateEvent").bind(update);

If I face the cube on the left, right, or backside the rotations are wrong.

I believe this is not a Lens Studio specific problem so similar solutions from other platforms like unity are welcome.