FIX: robots.txt

Make robots.txt just call Disallow on top-level directory on
website.  This will direct search engines to the sub directories.

Author: jklymak

robots.txt

@@ -5,32 +5,39 @@
 User-agent: *
 
 # do not search root directory by default.
-Disallow: /
-Allow: /_sitemap/
-
-# but allow some subdirectories:
-Allow: /stable/
-
-# versions should be searched.  Note sitemap.xml downweights them...
-Allow: /5.*/
-Allow: /4.*/
-Allow: /3.*/
-Allow: /2.*/
-Allow: /1.*/
-
-# packages that should still be searched:
-Allow: /cmocean/
-Allow: /mpl-probscale/
-Allow: /mpl_toolkits/
-Allow: /xkcd/
-Allow: /cycler/
-Allow: /matplotblog/
-Allow: /pytest-mpl/
-Allow: /basemap/
-Allow: /mpl-bench/
-Allow: /mpl-altair/
-Allow: /devdocs/
-
+Disallow: /api/
+Disallow: /devel/
+Disallow: /examples/
+Disallow: /faq/
+Disallow: /gallery/
+Disallow: /glossary/
+Disallow: /mpl_examples/
+Disallow: /mpl_toolkits/
+Disallow: /plot_directive/
+Disallow: /pyplots/
+Disallow: /resources/
+Disallow: /thirdpartypackages/
+Disallow: /tutorials/
+Disallow: /users/
+Disallow: /xkcd/
+Disallow: /_downloads/
+Disallow: /_images/
+Disallow: /_modules/
+Disallow: /_sources/
+Disallow: /_static/
+# files at top level:
+Disallow: /citing.html
+Disallow: /contents.html
+Disallow: /downloads.html
+Disallow: /gallery.html
+Disallow: /genindex.html
+Disallow: /index.html
+Disallow: /Matplotlib.pdf
+Disallow: /objects.inv
+Disallow: /py-modindex.html
+Disallow: /search.html
+Disallow: /searchindex.js
+Disallow: /win32_*.tar.gz
 
 # tell robots this is sitemap
 Sitemap: https://matplotlib.org/_sitemap/sitemap.xml