[www-doc] [Git][VideoLAN.org/websites][master] 2 commits: Generate the sitemap from the build

Jean-Baptiste Kempf (@jbk) gitlab at videolan.org
Tue Nov 10 12:37:41 UTC 2026



Jean-Baptiste Kempf pushed to branch master at VideoLAN organization / websites


Commits:
3e12a6af by Martin Finkel at 2026-11-10T13:36:24+01:00
Generate the sitemap from the build

sitemap-https.xml is generated on the server by a crawl of the docroot:
it lists test files, empty and forbidden pages, legacy slides, both
"dir/" and "dir/index.html" for the same page, and has no lastmod.

Generate sitemap-pages.xml at the end of the build instead, from the
pages built from PHP sources: English pages at their canonical URL,
translations at their own URL, each with its hreflang alternates, and
lastmod from the last commit of the source (and of the translation).

It gets a new name so the server cron cannot overwrite it, and
robots-https.txt now points to it.

- - - - -
bc552bd1 by Martin Finkel at 2026-11-10T13:36:24+01:00
sitemap: leave lastmod out without the full git history

When git cannot read the history, for example when the build user does
not own the checkout, or in a shallow clone, every file date is the
build date, and the sitemap claimed that every page changed today.
Leave lastmod out in that case and say so in the build log.

- - - - -


4 changed files:

- www.videolan.org/.gitignore
- www.videolan.org/Makefile.inc
- + www.videolan.org/bin/sitemap.php
- www.videolan.org/robots-https.txt


Changes:

=====================================
www.videolan.org/.gitignore
=====================================
@@ -8,6 +8,7 @@ vlc/download-*.php
 developers/i18n/images/*.png
 images/og/
 videolan/mirrors.txt
+sitemap-pages.xml
 js/slick.min.js
 style/slick.css
 style/slick.min.css


=====================================
www.videolan.org/Makefile.inc
=====================================
@@ -5,6 +5,14 @@ SUBDIR = doc security support vlc videolan projects developers locale press vlmc
 #PHP_FILES = # automatic
 PHP_IGNORES =
 
+# Sitemap of the built pages, once everything else is built
+all: sitemap-pages.xml
+
+sitemap-pages.xml: real-all bin/sitemap.php
+	@echo "Generating $@ ..."
+	php bin/sitemap.php > $@.tmp
+	mv $@.tmp $@
+
 # Additionnal dependencies
 
 news.html: news.msg


=====================================
www.videolan.org/bin/sitemap.php
=====================================
@@ -0,0 +1,128 @@
+<?php
+/*
+ * Generate the sitemap of www.videolan.org from the built pages.
+ *
+ * Usage, from the site root, after the pages are built:
+ *     php bin/sitemap.php > sitemap-pages.xml
+ *
+ * Lists every page generated from a PHP source: English pages at their
+ * canonical URL, translated pages at their own URL, each with its hreflang
+ * alternates. lastmod is the date of the last commit touching the source
+ * (and the translation for translated pages). Without the full git history,
+ * lastmod is left out: file dates would all be the build date.
+ */
+
+$site = 'https://www.videolan.org/';
+
+/* Not meant to be found from a search engine */
+$exclude = '#^(403|404|thank_you|news-rss|private/|templates/|stripe/|include/|vlc/download-skins2-go)#';
+
+$languages = array();
+foreach ( glob( 'locale/*', GLOB_ONLYDIR ) as $dir ) {
+    $languages[basename( $dir )] = true;
+}
+
+/* Date of the last commit of every file, from a single git log. A shallow
+ * clone would date every file to its few commits, so it counts as no history */
+$dates = array();
+$log = array();
+$shallow = trim( (string) shell_exec( 'git rev-parse --is-shallow-repository 2>/dev/null' ) );
+exec( 'git log --format=@%cs --name-only --relative -- . 2>/dev/null', $log, $status );
+if ( $shallow !== 'false' || $status !== 0 || empty( $log ) ) {
+    fwrite( STDERR, "sitemap.php: no full git history here (shallow: '$shallow', git exit $status), lastmod left out\n" );
+} else {
+    $date = null;
+    foreach ( $log as $line ) {
+        if ( $line === '' ) {
+            continue;
+        }
+        if ( $line[0] === '@' ) {
+            $date = substr( $line, 1 );
+        } elseif ( !isset( $dates[$line] ) ) {
+            $dates[$line] = $date;
+        }
+    }
+}
+
+function lastmod( $file ) {
+    global $dates;
+    return isset( $dates[$file] ) ? $dates[$file] : null;
+}
+
+function head_of( $html ) {
+    $s = file_get_contents( $html );
+    $end = strpos( $s, '</head>' );
+    return $end === false ? $s : substr( $s, 0, $end );
+}
+
+$entries = array();
+$files = new RecursiveIteratorIterator(
+    new RecursiveDirectoryIterator( '.', FilesystemIterator::SKIP_DOTS ) );
+foreach ( $files as $file ) {
+    $html = substr( $file->getPathname(), 2 );
+    if ( substr( $html, -5 ) !== '.html' || preg_match( $exclude, $html ) ) {
+        continue;
+    }
+    $base = substr( $html, 0, -5 );
+    $language = null;
+    if ( preg_match( '/^(.*)\.([a-zA-Z_]+)$/', $base, $m )
+         && isset( $languages[$m[2]] ) && is_file( $m[1].'.php' ) ) {
+        /* Translation generated from the English source (the generated
+         * index.xx.php may still be around) */
+        $source = $m[1].'.php';
+        $language = $m[2];
+    } elseif ( is_file( $base.'.php' ) ) {
+        $source = $base.'.php';
+    } else {
+        /* Static or leftover file */
+        continue;
+    }
+
+    $head = head_of( $html );
+    if ( preg_match( '/<meta name="robots" content="[^"]*noindex/i', $head ) ) {
+        continue;
+    }
+    if ( $language === null ) {
+        if ( !preg_match( '/<link rel="canonical" href="([^"]+)"/', $head, $m ) ) {
+            continue;
+        }
+        $url = $m[1];
+    } else {
+        $url = $site.$html;
+    }
+
+    $mod = lastmod( $source );
+    if ( $language !== null ) {
+        $po = lastmod( 'locale/'.$language.'/LC_MESSAGES/website.po' );
+        if ( $po > $mod ) {
+            $mod = $po;
+        }
+    }
+
+    $alternates = array();
+    if ( preg_match_all( '/<link rel="alternate" hreflang="([^"]+)" href="([^"]+)"/', $head, $m, PREG_SET_ORDER ) ) {
+        foreach ( $m as $alt ) {
+            $alternates[$alt[1]] = $alt[2];
+        }
+    }
+
+    $entries[$url] = array( $mod, $alternates );
+}
+ksort( $entries );
+
+echo '<?xml version="1.0" encoding="UTF-8"?>'."\n";
+echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"'."\n";
+echo '        xmlns:xhtml="http://www.w3.org/1999/xhtml">'."\n";
+foreach ( $entries as $url => $entry ) {
+    list( $mod, $alternates ) = $entry;
+    echo "<url>\n";
+    echo '  <loc>'.htmlspecialchars( $url, ENT_XML1 )."</loc>\n";
+    if ( $mod ) {
+        echo "  <lastmod>$mod</lastmod>\n";
+    }
+    foreach ( $alternates as $lang => $href ) {
+        echo '  <xhtml:link rel="alternate" hreflang="'.htmlspecialchars( $lang, ENT_XML1 ).'" href="'.htmlspecialchars( $href, ENT_XML1 ).'"/>'."\n";
+    }
+    echo "</url>\n";
+}
+echo "</urlset>\n";


=====================================
www.videolan.org/robots-https.txt
=====================================
@@ -1,6 +1,6 @@
 # $Id$
 
-Sitemap: https://www.videolan.org/sitemap-https.xml
+Sitemap: https://www.videolan.org/sitemap-pages.xml
 
 User-agent: *
 Disallow: /pub



View it on GitLab: https://code.videolan.org/VideoLAN.org/websites/-/compare/82f7ac614159b16805aaa40a10750d89454a175d...bc552bd13c89c75a617d4bea4e0f25b44675e8e2

-- 
View it on GitLab: https://code.videolan.org/VideoLAN.org/websites/-/compare/82f7ac614159b16805aaa40a10750d89454a175d...bc552bd13c89c75a617d4bea4e0f25b44675e8e2
You're receiving this email because of your account on code.videolan.org. Manage all notifications: https://code.videolan.org/-/profile/notifications | Help: https://code.videolan.org/help




More information about the www-doc mailing list