Array.map() returns an array. So for your example:
arrayTest.map(function (val) { return "<p>" + val + "</p>" })
will return: ["<p>a</p>","<p>b</p>"]; (with the comma in it).
You can use Array.join() to combine that array into a string without the comma: "<p>Header</p><p>a</p><p>b</p>"
arrayTest.map(function (val) { return "<p>" + val + "</p>" }).join("")
proof of concept fiddle

code snippet:
"use strict";
// This example displays a marker at the center of Australia.
// When the user clicks the marker, an info window opens.
function initMap() {
const uluru = {
lat: -25.363,
lng: 131.044,
};
const map = new google.maps.Map(document.getElementById("map"), {
zoom: 4,
center: uluru,
});
const infoWindow = new google.maps.InfoWindow();
var arrayTest = ["a", "b"];
infoWindow.setContent("<p>Header</p>" + arrayTest.map(function (val) { return "<p>" + val + "</p>" }).join(""));
const marker = new google.maps.Marker({
position: uluru,
map,
title: "Uluru (Ayers Rock)",
});
infoWindow.open(map, marker);
marker.addListener("click", () => {
infoWindow.open(map, marker);
});
}
/* Always set the map height explicitly to define the size of the div
* element that contains the map. */
#map {
height: 100%;
}
/* Optional: Makes the sample page fill the window. */
html,
body {
height: 100%;
margin: 0;
padding: 0;
}
<!DOCTYPE html>
<html>
<head>
<title>Info Windows</title>
<script src="https://polyfill.io/v3/polyfill.min.js?features=default"></script>
<script
src="https://maps.googleapis.com/maps/api/js?key=AIzaSyCkUOdZ5y7hMm0yrcCQoCvLwzdM6M8s5qk&callback=initMap&libraries=&v=weekly"
defer
></script>
<!-- jsFiddle will insert css and js -->
</head>
<body>
<div id="map"></div>
</body>
</html>