Get the text under highlight/squiggly/underline/strikethrough annotations (bug 1885505)

and add an invisible element containing the text in the annotation layer to make
it readable by a screen reader.

Author: calixteman

src/core/annotation.js

@@ -1246,6 +1246,10 @@ class Annotation {
     return null;
   }
 
+  get overlaysTextContent() {
+    return false;
+  }
+
   get hasTextContent() {
     return false;
   }
@@ -4691,6 +4695,10 @@ class HighlightAnnotation extends MarkupAnnotation {
     }
   }
 
+  get overlaysTextContent() {
+    return true;
+  }
+
   static createNewDict(annotation, xref, { apRef, ap }) {
     const { color, oldAnnotation, opacity, rect, rotation, user, quadPoints } =
       annotation;
@@ -4815,6 +4823,10 @@ class UnderlineAnnotation extends MarkupAnnotation {
       this.data.popupRef = null;
     }
   }
+
+  get overlaysTextContent() {
+    return true;
+  }
 }
 
 class SquigglyAnnotation extends MarkupAnnotation {
@@ -4859,6 +4871,10 @@ class SquigglyAnnotation extends MarkupAnnotation {
       this.data.popupRef = null;
     }
   }
+
+  get overlaysTextContent() {
+    return true;
+  }
 }
 
 class StrikeOutAnnotation extends MarkupAnnotation {
@@ -4898,6 +4914,10 @@ class StrikeOutAnnotation extends MarkupAnnotation {
       this.data.popupRef = null;
     }
   }
+
+  get overlaysTextContent() {
+    return true;
+  }
 }
 
 class StampAnnotation extends MarkupAnnotation {

src/core/document.js

@@ -66,6 +66,7 @@ import { calculateMD5 } from "./calculate_md5.js";
 import { Catalog } from "./catalog.js";
 import { clearGlobalCaches } from "./cleanup_helper.js";
 import { DatasetReader } from "./dataset_reader.js";
+import { Intersector } from "./intersector.js";
 import { Linearization } from "./parser.js";
 import { NullStream } from "./stream.js";
 import { ObjectLoader } from "./object_loader.js";
@@ -632,6 +633,7 @@ class Page {
     includeMarkedContent,
     disableNormalization,
     sink,
+    intersector = null,
   }) {
     const contentStreamPromise = this.getContentStream();
     const resourcesPromise = this.loadResources(RESOURCES_KEYS_TEXT_CONTENT);
@@ -658,6 +660,7 @@ class Page {
       sink,
       viewBox: this.view,
       lang,
+      intersector,
     });
   }
 
@@ -707,6 +710,8 @@ class Page {
       intentDisplay = !!(intent & RenderingIntentFlag.DISPLAY),
       intentPrint = !!(intent & RenderingIntentFlag.PRINT);
 
+    const highlightedAnnotations = [];
+
     for (const annotation of annotations) {
       // Get the annotation even if it's hidden because
       // JS can change its display.
@@ -732,9 +737,29 @@ class Page {
               );
             })
         );
+      } else if (annotation.overlaysTextContent && isVisible) {
+        highlightedAnnotations.push(annotation);
       }
     }
 
+    if (highlightedAnnotations.length > 0) {
+      const intersector = new Intersector(highlightedAnnotations);
+      textContentPromises.push(
+        this.extractTextContent({
+          handler,
+          task,
+          includeMarkedContent: false,
+          disableNormalization: false,
+          sink: null,
+          viewBox: this.view,
+          lang: null,
+          intersector,
+        }).then(() => {
+          intersector.setText();
+        })
+      );
+    }
+
     await Promise.all(textContentPromises);
     return annotationsData;
   }

src/core/evaluator.js

@@ -2362,6 +2362,7 @@ class PartialEvaluator {
     disableNormalization = false,
     keepWhiteSpace = false,
     prevRefs = null,
+    intersector = null,
   }) {
     const objId = stream.dict?.objId;
     const seenRefs = new RefSet(prevRefs);
@@ -2506,6 +2507,7 @@ class PartialEvaluator {
       transform = textContentItem.prevTransform,
       fontName = textContentItem.fontName,
     }) {
+      intersector?.addExtraChar(" ");
       textContent.items.push({
         str: " ",
         dir: "ltr",
@@ -2964,9 +2966,21 @@ class PartialEvaluator {
 
         if (!font.vertical) {
           scaledDim *= textState.textHScale;
+          intersector?.addGlyph(
+            getCurrentTextTransform(),
+            scaledDim,
+            0,
+            glyph.unicode
+          );
           textState.translateTextMatrix(scaledDim, 0);
           textChunk.width += scaledDim;
         } else {
+          intersector?.addGlyph(
+            getCurrentTextTransform(),
+            0,
+            scaledDim,
+            glyph.unicode
+          );
           textState.translateTextMatrix(0, scaledDim);
           scaledDim = Math.abs(scaledDim);
           textChunk.height += scaledDim;
@@ -2985,8 +2999,12 @@ class PartialEvaluator {
           // alignment issues between the textLayer and the canvas if the text
           // contains e.g. tabs (fixes issue6612.pdf).
           textChunk.str.push(" ");
+          intersector?.addExtraChar(" ");
+        }
+
+        if (!intersector) {
+          textChunk.str.push(glyphUnicode);
         }
-        textChunk.str.push(glyphUnicode);
 
         if (charSpacing) {
           if (!font.vertical) {
@@ -3002,6 +3020,7 @@ class PartialEvaluator {
     }
 
     function appendEOL() {
+      intersector?.addExtraChar("\n");
       resetLastChars();
       if (textContentItem.initialized) {
         textContentItem.hasEOL = true;
@@ -3027,6 +3046,7 @@ class PartialEvaluator {
         if (textContentItem.initialized) {
           resetLastChars();
           textContentItem.str.push(" ");
+          intersector?.addExtraChar(" ");
         }
         return false;
       }
@@ -3078,7 +3098,7 @@ class PartialEvaluator {
       if (batch && length < TEXT_CHUNK_BATCH_SIZE) {
         return;
       }
-      sink.enqueue(textContent, length);
+      sink?.enqueue(textContent, length);
       textContent.items = [];
       textContent.styles = Object.create(null);
     }
@@ -3088,7 +3108,7 @@ class PartialEvaluator {
     return new Promise(function promiseBody(resolve, reject) {
       const next = function (promise) {
         enqueueChunk(/* batch = */ true);
-        Promise.all([promise, sink.ready]).then(function () {
+        Promise.all([promise, sink?.ready]).then(function () {
           try {
             promiseBody(resolve, reject);
           } catch (ex) {
@@ -3341,7 +3361,7 @@ class PartialEvaluator {
                   },
 
                   get desiredSize() {
-                    return sink.desiredSize;
+                    return sink.desiredSize ?? 0;
                   },
 
                   get ready() {
@@ -3359,7 +3379,7 @@ class PartialEvaluator {
                         : resources,
                     stateManager: xObjStateManager,
                     includeMarkedContent,
-                    sink: sinkWrapper,
+                    sink: sink && sinkWrapper,
                     seenStyles,
                     viewBox,
                     lang,
@@ -3499,7 +3519,7 @@ class PartialEvaluator {
             }
             break;
         } // switch
-        if (textContent.items.length >= sink.desiredSize) {
+        if (textContent.items.length >= (sink?.desiredSize ?? 1)) {
           // Wait for ready, if we reach highWaterMark.
           stop = true;
           break;