🗺️ webtui: generate real sitemap.xml (160 URLs) + robots.txt — fixes could-not-fetch for search engines
swadhinbiswas committed 2 months ago
a225470
1 parent 0696dbc
commit a225470
1 file changed
+23 -0
Showing 1 changed file+23-0
▼webtui/build.py
+23-0
| @@ -188,8 +188,31 @@ def build() -> None: | ||
188 | 188 | landing = landing.replace("{{ toc }}", "") |
189 | 189 | (DIST / "index.html").write_text(landing) |
190 | 190 | |
191 | + # ─── Sitemap + robots (fixes "could not fetch sitemap" for search engines) ─── | |
192 | + SITE = "https://justapi.pages.dev" | |
193 | + urls = [f"<url><loc>{SITE}/</loc></url>"] | |
194 | + for md_file in sorted(CONTENT.rglob("*.md*")): | |
195 | + rel = md_file.relative_to(CONTENT) | |
196 | + if len(rel.parts) == 1: | |
197 | + continue | |
198 | + path = "/" + str(rel.with_suffix("")).replace(chr(92), "/") + "/" | |
199 | + urls.append(f"<url><loc>{SITE}{path}</loc></url>") | |
200 | + sitemap = ( | |
201 | + '<?xml version="1.0" encoding="UTF-8"?>\n' | |
202 | + '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n' | |
203 | + + "\n".join(urls) | |
204 | + + "\n</urlset>\n" | |
205 | + ) | |
206 | + (DIST / "sitemap.xml").write_text(sitemap) | |
207 | + (DIST / "robots.txt").write_text( | |
208 | + "User-agent: *\nAllow: /\n\nSitemap: " + SITE + "/sitemap.xml\n" | |
209 | + ) | |
210 | + | |
191 | 211 | print(f"built {count} docs pages + landing -> {DIST}") |
192 | 212 | |
213 | + # sitemap has 1 (landing) + count URLs | |
214 | + print(f"sitemap.xml: {len(urls)} URLs") | |
215 | + | |
193 | 216 | |
194 | 217 | if __name__ == "__main__": |
195 | 218 | build() |
196 | 219 | |