Add RSS generation (#123)

6 files changed, 171 insertions, 25 deletions

diff --git a/src/@types/astro.ts b/src/@types/astro.ts
index d991dbbcf..049105970 100644
--- a/src/@types/astro.ts
+++ b/src/@types/astro.ts
@@ -14,13 +14,16 @@ export interface AstroConfig {
   astroRoot: URL;
   public: URL;
   extensions?: Record<string, ValidExtensionPlugins>;
-  /** Public URL base (e.g. 'https://mysite.com'). Used in generating sitemaps and canonical URLs. */
-  site?: string;
-  /** Generate a sitemap? */
+  /** Options specific to `astro build` */
   buildOptions: {
+    /** Your public domain, e.g.: https://my-site.dev/. Used to generate sitemaps and canonical URLs. */
+    site?: string;
+    /** Generate sitemap (set to "false" to disable) */
     sitemap: boolean;
   };
+  /** Options for the development server run with `astro dev`. */
   devOptions: {
+    /** The port to run the dev server on. */
     port: number;
     projectRoot?: string;
   };
@@ -34,7 +37,7 @@ export type AstroUserConfig = Omit<AstroConfig, 'buildOptions' | 'devOptions'> &
     port?: number;
     projectRoot?: string;
   };
-}
+};
 
 export interface JsxItem {
   name: string;
@@ -67,6 +70,34 @@ export interface CreateCollection<T = any> {
   permalink?: ({ params }: { params: Params }) => string;
   /** page size */
   pageSize?: number;
+  /** Generate RSS feed from data() */
+  rss?: CollectionRSS<T>;
+}
+
+export interface CollectionRSS<T = any> {
+  /** (required) Title of the RSS Feed */
+  title: string;
+  /** (required) Description of the RSS Feed */
+  description: string;
+  /** Specify arbitrary metadata on opening <xml> tag */
+  xmlns?: Record<string, string>;
+  /** Specify custom data in opening of file */
+  customData?: string;
+  /** Return data about each item */
+  item: (
+    item: T
+  ) => {
+    /** (required) Title of item */
+    title: string;
+    /** (required) Link to item */
+    link: string;
+    /** Publication date of item */
+    pubDate?: Date;
+    /** Item description */
+    description?: string;
+    /** Append some other XML-valid data to this item */
+    customData?: string;
+  };
 }
 
 export interface CollectionResult<T = any> {
diff --git a/src/build.ts b/src/build.ts
index 8310b1179..2c61b3fed 100644
--- a/src/build.ts
+++ b/src/build.ts
@@ -10,8 +10,10 @@ import { fdir } from 'fdir';
 import { defaultLogDestination, error, info } from './logger.js';
 import { createRuntime } from './runtime.js';
 import { bundle, collectDynamicImports } from './build/bundle.js';
+import { generateRSS } from './build/rss.js';
 import { generateSitemap } from './build/sitemap.js';
 import { collectStatics } from './build/static.js';
+import { canonicalURL } from './build/util.js';
 
 const { mkdir, readFile, writeFile } = fsPromises;
 
@@ -20,12 +22,14 @@ interface PageBuildOptions {
   dist: URL;
   filepath: URL;
   runtime: AstroRuntime;
+  site?: string;
   sitemap: boolean;
   statics: Set<string>;
 }
 
 interface PageResult {
   canonicalURLs: string[];
+  rss?: string;
   statusCode: number;
 }
 
@@ -78,7 +82,7 @@ function getPageType(filepath: URL): 'collection' | 'static' {
 }
 
 /** Build collection */
-async function buildCollectionPage({ astroRoot, dist, filepath, runtime, statics }: PageBuildOptions): Promise<PageResult> {
+async function buildCollectionPage({ astroRoot, dist, filepath, runtime, site, statics }: PageBuildOptions): Promise<PageResult> {
   const rel = path.relative(fileURLToPath(astroRoot) + '/pages', fileURLToPath(filepath)); // pages/index.astro
   const pagePath = `/${rel.replace(/\$([^.]+)\.astro$/, '$1')}`;
   const builtURLs = new Set<string>(); // !important: internal cache that prevents building the same URLs
@@ -97,12 +101,19 @@ async function buildCollectionPage({ astroRoot, dist, filepath, runtime, statics
   }
 
   const result = (await loadCollection(pagePath)) as LoadResult;
+
+  if (result.statusCode >= 500) {
+    throw new Error((result as any).error);
+  }
   if (result.statusCode === 200 && !result.collectionInfo) {
     throw new Error(`[${rel}]: Collection page must export createCollection() function`);
   }
 
+  let rss: string | undefined;
+
   // note: for pages that require params (/tag/:tag), we will get a 404 but will still get back collectionInfo that tell us what the URLs should be
   if (result.collectionInfo) {
+    // build subsequent pages
     await Promise.all(
       [...result.collectionInfo.additionalURLs].map(async (url) => {
         // for the top set of additional URLs, we render every new URL generated
@@ -114,11 +125,17 @@ async function buildCollectionPage({ astroRoot, dist, filepath, runtime, statics
         }
       })
     );
+
+    if (result.collectionInfo.rss) {
+      if (!site) throw new Error(`[${rel}] createCollection() tried to generate RSS but "buildOptions.site" missing in astro.config.mjs`);
+      rss = generateRSS({ ...(result.collectionInfo.rss as any), site }, rel.replace(/\$([^.]+)\.astro$/, '$1'));
+    }
   }
 
   return {
     canonicalURLs: [...builtURLs].filter((url) => !url.endsWith('/1')), // note: canonical URLs are controlled by the collection, so these are canonical (but exclude "/1" pages as those are duplicates of the index)
     statusCode: result.statusCode,
+    rss,
   };
 }
 
@@ -186,10 +203,17 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
         const filepath = new URL(`file://${pathname}`);
 
         const pageType = getPageType(filepath);
-        const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, sitemap: astroConfig.buildOptions.sitemap, statics };
+        const pageOptions: PageBuildOptions = { astroRoot, dist, filepath, runtime, site: astroConfig.buildOptions.site, sitemap: astroConfig.buildOptions.sitemap, statics };
         if (pageType === 'collection') {
-          const { canonicalURLs } = await buildCollectionPage(pageOptions);
+          const { canonicalURLs, rss } = await buildCollectionPage(pageOptions);
           builtURLs.push(...canonicalURLs);
+          if (rss) {
+            const basename = path
+              .relative(fileURLToPath(astroRoot) + '/pages', pathname)
+              .replace(/^\$/, '')
+              .replace(/\.astro$/, '');
+            await writeFilep(new URL(`file://${path.join(fileURLToPath(dist), 'feed', basename + '.xml')}`), rss, 'utf8');
+          }
         } else {
           const { canonicalURLs } = await buildStaticPage(pageOptions);
           builtURLs.push(...canonicalURLs);
@@ -239,18 +263,11 @@ export async function build(astroConfig: AstroConfig): Promise<0 | 1> {
   }
 
   // build sitemap
-  if (astroConfig.buildOptions.sitemap && astroConfig.site) {
-    const sitemap = generateSitemap(
-      builtURLs.map((url) => ({
-        canonicalURL: new URL(
-          path.extname(url) ? url : url.replace(/\/?$/, '/'), // add trailing slash if there’s no extension
-          astroConfig.site
-        ).href,
-      }))
-    );
+  if (astroConfig.buildOptions.sitemap && astroConfig.buildOptions.site) {
+    const sitemap = generateSitemap(builtURLs.map((url) => ({ canonicalURL: canonicalURL(url, astroConfig.buildOptions.site) })));
     await writeFile(new URL('./sitemap.xml', dist), sitemap, 'utf8');
   } else if (astroConfig.buildOptions.sitemap) {
-    info(logging, 'tip', `Set your "site" in astro.config.mjs to generate a sitemap.xml`);
+    info(logging, 'tip', `Set "buildOptions.site" in astro.config.mjs to generate a sitemap.xml`);
   }
 
   await runtime.shutdown();
diff --git a/src/build/rss.ts b/src/build/rss.ts
new file mode 100644
index 000000000..b75ed908b
--- /dev/null
+++ b/src/build/rss.ts
@@ -0,0 +1,68 @@
+import type { CollectionRSS } from '../@types/astro';
+import parser from 'fast-xml-parser';
+import { canonicalURL } from './util.js';
+
+/** Validates createCollection.rss */
+export function validateRSS(rss: CollectionRSS, filename: string): void {
+  if (!rss.title) throw new Error(`[${filename}] rss.title required`);
+  if (!rss.description) throw new Error(`[${filename}] rss.description required`);
+  if (typeof rss.item !== 'function') throw new Error(`[${filename}] rss.item() function required`);
+}
+
+/** Generate RSS 2.0 feed */
+export function generateRSS<T>(input: { data: T[]; site: string } & CollectionRSS<T>, filename: string): string {
+  let xml = `<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"`;
+
+  validateRSS(input as any, filename);
+
+  // xmlns
+  if (input.xmlns) {
+    for (const [k, v] of Object.entries(input.xmlns)) {
+      xml += ` xmlns:${k}="${v}"`;
+    }
+  }
+  xml += `>`;
+  xml += `<channel>`;
+
+  // title, description, customData
+  xml += `<title><![CDATA[${input.title}]]></title>`;
+  xml += `<description><![CDATA[${input.description}]]></description>`;
+  xml += `<link>${canonicalURL('/feed/' + filename + '.xml', input.site)}</link>`;
+  if (typeof input.customData === 'string') xml += input.customData;
+
+  // items
+  if (!Array.isArray(input.data) || !input.data.length) throw new Error(`[${filename}] data() returned no items. Can’t generate RSS feed.`);
+  for (const item of input.data) {
+    xml += `<item>`;
+    const result = input.item(item);
+    // validate
+    if (typeof result !== 'object') throw new Error(`[${filename}] rss.item() expected to return an object, returned ${typeof result}.`);
+    if (!result.title) throw new Error(`[${filename}] rss.item() returned object but required "title" is missing.`);
+    if (!result.link) throw new Error(`[${filename}] rss.item() returned object but required "link" is missing.`);
+    xml += `<title><![CDATA[${result.title}]]></title>`;
+    xml += `<link>${canonicalURL(result.link, input.site)}</link>`;
+    if (result.description) xml += `<description><![CDATA[${result.description}]]></description>`;
+    if (result.pubDate) {
+      // note: this should be a Date, but if user provided a string or number, we can work with that, too.
+      if (typeof result.pubDate === 'number' || typeof result.pubDate === 'string') {
+        result.pubDate = new Date(result.pubDate);
+      } else if (result.pubDate instanceof Date === false) {
+        throw new Error('[${filename}] rss.item().pubDate must be a Date');
+      }
+      xml += `<pubDate>${result.pubDate.toUTCString()}</pubDate>`;
+    }
+    if (typeof result.customData === 'string') xml += result.customData;
+    xml += `</item>`;
+  }
+
+  xml += `</channel></rss>`;
+
+  // validate user’s inputs to see if it’s valid XML
+  const isValid = parser.validate(xml);
+  if (isValid !== true) {
+    // If valid XML, isValid will be `true`. Otherwise, this will be an error object. Throw.
+    throw new Error(isValid as any);
+  }
+
+  return xml;
+}
diff --git a/src/build/static.ts b/src/build/static.ts
index 96cb72b7f..af99c33cb 100644
--- a/src/build/static.ts
+++ b/src/build/static.ts
@@ -10,7 +10,7 @@ export function collectStatics(html: string) {
   const append = (el: Element, attr: string) => {
     // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
     const value: string = $(el).attr(attr)!;
-    if (value.startsWith('http')) {
+    if (value.startsWith('http') || $(el).attr('rel') === 'alternate') {
       return;
     }
     statics.add(value);
diff --git a/src/build/util.ts b/src/build/util.ts
new file mode 100644
index 000000000..505e6f183
--- /dev/null
+++ b/src/build/util.ts
@@ -0,0 +1,9 @@
+import path from 'path';
+
+/** Normalize URL to its canonical form */
+export function canonicalURL(url: string, base?: string): string {
+  return new URL(
+    path.extname(url) ? url : url.replace(/(\/+)?$/, '/'), // add trailing slash if there’s no extension
+    base
+  ).href;
+}
diff --git a/src/runtime.ts b/src/runtime.ts
index 6a4702a78..be609ed0a 100644
--- a/src/runtime.ts
+++ b/src/runtime.ts
@@ -1,6 +1,6 @@
 import { fileURLToPath } from 'url';
 import type { SnowpackDevServer, ServerRuntime as SnowpackServerRuntime, SnowpackConfig } from 'snowpack';
-import type { AstroConfig, CollectionResult, CreateCollection, Params, RuntimeMode } from './@types/astro';
+import type { AstroConfig, CollectionResult, CollectionRSS, CreateCollection, Params, RuntimeMode } from './@types/astro';
 import type { LogOptions } from './logger';
 import type { CompileError } from './parser/utils/error.js';
 import { debug, info } from './logger.js';
@@ -26,7 +26,10 @@ interface RuntimeConfig {
 }
 
 // info needed for collection generation
-type CollectionInfo = { additionalURLs: Set<string> };
+interface CollectionInfo {
+  additionalURLs: Set<string>;
+  rss?: { data: any[] & CollectionRSS };
+}
 
 type LoadResultSuccess = {
   statusCode: 200;
@@ -78,6 +81,7 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
   }
 
   const snowpackURL = searchResult.location.snowpackURL;
+  let rss: { data: any[] & CollectionRSS } = {} as any;
 
   try {
     const mod = await backendSnowpackRuntime.importModule(snowpackURL);
@@ -90,11 +94,11 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
     if (mod.exports.createCollection) {
       const createCollection: CreateCollection = await mod.exports.createCollection();
       for (const key of Object.keys(createCollection)) {
-        if (key !== 'data' && key !== 'routes' && key !== 'permalink' && key !== 'pageSize') {
+        if (key !== 'data' && key !== 'routes' && key !== 'permalink' && key !== 'pageSize' && key !== 'rss') {
           throw new Error(`[createCollection] unknown option: "${key}"`);
         }
       }
-      let { data: loadData, routes, permalink, pageSize } = createCollection;
+      let { data: loadData, routes, permalink, pageSize, rss: createRSS } = createCollection;
       if (!pageSize) pageSize = 25; // can’t be 0
       let currentParams: Params = {};
 
@@ -116,6 +120,14 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
 
       let data: any[] = await loadData({ params: currentParams });
 
+      // handle RSS
+      if (createRSS) {
+        rss = {
+          ...createRSS,
+          data: [...data] as any,
+        };
+      }
+
       collection.start = 0;
       collection.end = data.length - 1;
       collection.total = data.length;
@@ -161,7 +173,10 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
         return {
           statusCode: 301,
           location: reqPath + '/1',
-          collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
+          collectionInfo: {
+            additionalURLs,
+            rss: rss.data ? rss : undefined,
+          },
         };
       }
 
@@ -170,7 +185,10 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
         return {
           statusCode: 404,
           error: new Error('Not Found'),
-          collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
+          collectionInfo: {
+            additionalURLs,
+            rss: rss.data ? rss : undefined,
+          },
         };
       }
 
@@ -200,7 +218,10 @@ async function load(config: RuntimeConfig, rawPathname: string | undefined): Pro
       statusCode: 200,
       contentType: 'text/html; charset=utf-8',
       contents: html,
-      collectionInfo: additionalURLs.size ? { additionalURLs } : undefined,
+      collectionInfo: {
+        additionalURLs,
+        rss: rss.data ? rss : undefined,
+      },
     };
   } catch (err) {
     if (err.code === 'parse-error') {