ENH: add sitemap to robots.txt

Author: jklymak

robots.txt

@@ -1,13 +1,27 @@
+# Docs: https://developers.google.com/search/docs/advanced/robots/intro
+# Note old files will still be indexed if they have links to them,
+# hopefully they are weighted less...
+
 User-agent: *
+
+# do not search root directory by default.
 Disallow: /
-Disallow: /examples/  # outdated location. Current examples are in /gallery/.
-Allow: /cmocean/
-Allow: /mpl-probscale/
-Allow: /mpl_toolkits/
-Allow: /xkcd/
+
+# but allow some subdirectories:
 Allow: /stable/
+
+# versions should be searched.  Note sitemap.xml downweights them...
 Allow: /5.*/
 Allow: /4.*/
 Allow: /3.*/
 Allow: /2.*/
 Allow: /1.*/
+
+# packages that should still be searched:
+Allow: /cmocean/
+Allow: /mpl-probscale/
+Allow: /mpl_toolkits/
+Allow: /xkcd/
+
+# tell robots this is sitemap
+Sitemap: https://matplotlib.org/sitemap.xml