Blog Support 1/3: Data fetching (#62)

* Add example blog

* Add author data

* Improve navigation

* Style nav

* Add friendly error message

* Throw error if import glob used for non-Markdown files

* Use import.meta.collection() API instead

* README fixes

2 files changed, 112 insertions, 16 deletions

diff --git a/src/compiler/codegen.ts b/src/compiler/codegen.ts
index e64051317..59cc2c702 100644
--- a/src/compiler/codegen.ts
+++ b/src/compiler/codegen.ts
@@ -5,6 +5,7 @@ import type { JsxItem, TransformResult } from '../@types/astro';
 
 import eslexer from 'es-module-lexer';
 import esbuild from 'esbuild';
+import glob from 'tiny-glob/sync.js';
 import path from 'path';
 import { walk } from 'estree-walker';
 import babelParser from '@babel/parser';
@@ -35,6 +36,15 @@ function internalImport(internalPath: string) {
   return `/_astro_internal/${internalPath}`;
 }
 
+/** Is this an import.meta.* built-in? You can pass an optional 2nd param to see if the name matches as well. */
+function isImportMetaDeclaration(declaration: VariableDeclarator, metaName?: string): boolean {
+  const { init } = declaration;
+  if (!init || init.type !== 'CallExpression' || init.callee.type !== 'MemberExpression' || init.callee.object.type !== 'MetaProperty') return false;
+  // optional: if metaName specified, match that
+  if (metaName && (init.callee.property.type !== 'Identifier' || init.callee.property.name !== metaName)) return false;
+  return true;
+}
+
 /** Retrieve attributes from TemplateNode */
 function getAttributes(attrs: Attribute[]): Record<string, string> {
   let result: Record<string, string> = {};
@@ -272,6 +282,10 @@ interface CodegenState {
   dynamicImports: DynamicImportMap;
 }
 
+// cache filesystem pings
+const miniGlobCache = new Map<string, Map<string, string[]>>();
+
+/** Compile/prepare Astro frontmatter scripts */
 function compileModule(module: Script, state: CodegenState, compileOptions: CompileOptions) {
   const { extensions = defaultExtensions } = compileOptions;
 
@@ -279,8 +293,11 @@ function compileModule(module: Script, state: CodegenState, compileOptions: Comp
   const componentProps: VariableDeclarator[] = [];
   const componentExports: ExportNamedDeclaration[] = [];
 
+  const collectionImports = new Map<string, string>();
+
   let script = '';
   let propsStatement = '';
+  let dataStatement = '';
   const componentPlugins = new Set<ValidExtensionPlugins>();
 
   if (module) {
@@ -293,12 +310,17 @@ function compileModule(module: Script, state: CodegenState, compileOptions: Comp
     let i = body.length;
     while (--i >= 0) {
       const node = body[i];
-      if (node.type === 'ImportDeclaration') {
-        componentImports.push(node);
-        body.splice(i, 1);
-      }
-      if (/^Export/.test(node.type)) {
-        if (node.type === 'ExportNamedDeclaration' && node.declaration?.type === 'VariableDeclaration') {
+      switch (node.type) {
+        case 'ImportDeclaration': {
+          componentImports.push(node);
+          body.splice(i, 1); // remove node
+          break;
+        }
+        case 'ExportNamedDeclaration': {
+          if (node.declaration?.type !== 'VariableDeclaration') {
+            // const replacement = extract_exports(node);
+            break;
+          }
           const declaration = node.declaration.declarations[0];
           if ((declaration.id as Identifier).name === '__layout' || (declaration.id as Identifier).name === '__content') {
             componentExports.push(node);
@@ -306,8 +328,31 @@ function compileModule(module: Script, state: CodegenState, compileOptions: Comp
             componentProps.push(declaration);
           }
           body.splice(i, 1);
+          break;
+        }
+        case 'VariableDeclaration': {
+          for (const declaration of node.declarations) {
+            // only select import.meta.collection() calls here. this utility filters those out for us.
+            if (!isImportMetaDeclaration(declaration, 'collection')) continue;
+            if (declaration.id.type !== 'Identifier') continue;
+            const { id, init } = declaration;
+            if (!id || !init || init.type !== 'CallExpression') continue;
+
+            // gather data
+            const namespace = id.name;
+
+            // TODO: support more types (currently we can; it’s just a matter of parsing out the expression)
+            if ((init as any).arguments[0].type !== 'StringLiteral') {
+              throw new Error(`[import.meta.collection] Only string literals allowed, ex: \`import.meta.collection('./post/*.md')\`\n  ${state.filename}`);
+            }
+            const spec = (init as any).arguments[0].value;
+            if (typeof spec === 'string') collectionImports.set(namespace, spec);
+
+            // remove node
+            body.splice(i, 1);
+          }
+          break;
         }
-        // const replacement = extract_exports(node);
       }
     }
 
@@ -339,14 +384,65 @@ function compileModule(module: Script, state: CodegenState, compileOptions: Comp
         }
         propsStatement += `,`;
       }
-      propsStatement += `} = props;`;
+      propsStatement += `} = props;\n`;
     }
-    script = propsStatement + babelGenerator(program).code;
+
+    // handle importing data
+    for (const [namespace, spec] of collectionImports.entries()) {
+      // only allow for .md files
+      if (!spec.endsWith('.md')) {
+        throw new Error(`Only *.md pages are supported for import.meta.collection(). Attempted to load "${spec}"`);
+      }
+
+      // locate files
+      try {
+        let found: string[];
+
+        // use cache
+        let cachedLookups = miniGlobCache.get(state.filename);
+        if (!cachedLookups) {
+          cachedLookups = new Map();
+          miniGlobCache.set(state.filename, cachedLookups);
+        }
+        if (cachedLookups.get(spec)) {
+          found = cachedLookups.get(spec) as string[];
+        } else {
+          found = glob(spec, { cwd: path.dirname(state.filename), filesOnly: true });
+          cachedLookups.set(spec, found);
+          miniGlobCache.set(state.filename, cachedLookups);
+        }
+
+        // throw error, purge cache if no results found
+        if (!found.length) {
+          cachedLookups.delete(spec);
+          miniGlobCache.set(state.filename, cachedLookups);
+          throw new Error(`No files matched "${spec}" from ${state.filename}`);
+        }
+
+        const data = found.map((importPath) => {
+          if (importPath.startsWith('http') || importPath.startsWith('.')) return importPath;
+          return `./` + importPath;
+        });
+
+        // add static imports (probably not the best, but async imports don‘t work just yet)
+        data.forEach((importPath, j) => {
+          state.importExportStatements.add(`const ${namespace}_${j} = import('${importPath}').then((m) => ({ ...m.__content, url: '${importPath.replace(/\.md$/, '')}' }));`);
+        });
+
+        // expose imported data to Astro script
+        dataStatement += `const ${namespace} = await Promise.all([${found.map((_, j) => `${namespace}_${j}`).join(',')}]);\n`;
+      } catch (err) {
+        throw new Error(`No files matched "${spec}" from ${state.filename}`);
+      }
+    }
+
+    script = propsStatement + dataStatement + babelGenerator(program).code;
   }
 
   return { script, componentPlugins };
 }
 
+/** Compile styles */
 function compileCss(style: Style, state: CodegenState) {
   walk(style, {
     enter(node: TemplateNode) {
@@ -363,6 +459,7 @@ function compileCss(style: Style, state: CodegenState) {
   });
 }
 
+/** Compile page markup */
 function compileHtml(enterNode: TemplateNode, state: CodegenState, compileOptions: CompileOptions) {
   const { components, css, importExportStatements, dynamicImports, filename } = state;
   const { astroConfig } = compileOptions;
diff --git a/src/config.ts b/src/config.ts
index fe4549929..70463fee7 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -35,9 +35,10 @@ function configDefaults(userConfig?: any): any {
 function normalizeConfig(userConfig: any, root: string): AstroConfig {
   const config: any = { ...(userConfig || {}) };
 
-  config.projectRoot = new URL(config.projectRoot + '/', root);
-  config.astroRoot = new URL(config.astroRoot + '/', root);
-  config.public = new URL(config.public + '/', root);
+  const fileProtocolRoot = `file://${root}/`;
+  config.projectRoot = new URL(config.projectRoot + '/', fileProtocolRoot);
+  config.astroRoot = new URL(config.astroRoot + '/', fileProtocolRoot);
+  config.public = new URL(config.public + '/', fileProtocolRoot);
 
   return config as AstroConfig;
 }
@@ -48,13 +49,11 @@ export async function loadConfig(rawRoot: string | undefined): Promise<AstroConf
     rawRoot = process.cwd();
   }
 
-  let config: any;
-
   const root = pathResolve(rawRoot);
-  const fileProtocolRoot = `file://${root}/`;
   const astroConfigPath = pathJoin(root, 'astro.config.mjs');
 
   // load
+  let config: any;
   if (existsSync(astroConfigPath)) {
     config = configDefaults((await import(astroConfigPath)).default);
   } else {
@@ -65,7 +64,7 @@ export async function loadConfig(rawRoot: string | undefined): Promise<AstroConf
   validateConfig(config);
 
   // normalize
-  config = normalizeConfig(config, fileProtocolRoot);
+  config = normalizeConfig(config, root);
 
   return config as AstroConfig;
 }