Another approach...
Map the characters in the string as having a value of 1 for '(' and -1 for ')'.
Scan to get cumulative sums of the mapped values.
If all the cumulative sums are >= 0 and the last sum is = 0, the nesting is correct.

let isNested (chs : string) = 

    let scores = 
        (chs.ToCharArray())
        |> Seq.map (fun ch -> 
                        match ch with
                        |   '(' -> 1
                        |   ')' -> -1
                        |   _   -> 0)
        |>  Seq.scan (fun counter value -> counter + value) 0
        |>  Seq.cache

    scores 
    |> Seq.forall (fun counter -> counter >= 0)
    && 
    scores 
    |> Seq.last = 0

Another approach...
Map the characters in the string as having a value of 1 for '(' and -1 for ')'.
Scan to get cumulative sums of the mapped values.
If all the cumulative sums are >= 0 and the last sum is = 0, the nesting is correct.

let isNested (chs : string) = 

    let scores = 
        (chs.ToCharArray())
        |> Seq.map (fun ch -> 
                        match ch with
                        |   '(' -> 1
                        |   ')' -> -1
                        |   _   -> 0)
        |>  Seq.scan (fun counter value -> counter + value) 0 // or |>  Seq.scan (+) 0
        |>  Seq.cache

    scores 
    |> Seq.forall (fun counter -> counter >= 0)
    && 
    scores 
    |> Seq.last = 0