diff options
28 files changed, 938 insertions, 35 deletions
diff --git a/.changeset/lazy-phones-run.md b/.changeset/lazy-phones-run.md new file mode 100644 index 000000000..906dbda2b --- /dev/null +++ b/.changeset/lazy-phones-run.md @@ -0,0 +1,6 @@ +--- +'@astrojs/telemetry': minor +'astro': patch +--- + +Adds anonymous telemetry data to the cli diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56aff2e63..bb9d6cbd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,6 +109,8 @@ jobs: test: name: 'Test: ${{ matrix.os }} (node@${{ matrix.node_version }})' runs-on: ${{ matrix.os }} + env: + ASTRO_TELEMETRY_DISABLED: true strategy: matrix: os: [ubuntu-latest] diff --git a/packages/astro/package.json b/packages/astro/package.json index 6167d2155..9952a36a6 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -79,6 +79,7 @@ "@astrojs/language-server": "^0.13.4", "@astrojs/markdown-remark": "^0.9.2", "@astrojs/prism": "0.4.1", + "@astrojs/telemetry": "^0.0.1", "@astrojs/webapi": "^0.11.1", "@babel/core": "^7.17.9", "@babel/generator": "^7.17.9", diff --git a/packages/astro/src/cli/index.ts b/packages/astro/src/cli/index.ts index f4a7ed895..ca43386da 100644 --- a/packages/astro/src/cli/index.ts +++ b/packages/astro/src/cli/index.ts @@ -6,6 +6,9 @@ import { LogOptions } from '../core/logger/core.js'; import * as colors from 'kleur/colors'; import yargs from 'yargs-parser'; import { z } from 'zod'; +import { AstroTelemetry } from '@astrojs/telemetry'; +import * as event from '@astrojs/telemetry/events'; + import { nodeLogDestination, enableVerboseLogging } from '../core/logger/node.js'; import build from '../core/build/index.js'; import add from '../core/add/index.js'; @@ -13,6 +16,7 @@ import devServer from '../core/dev/index.js'; import preview from '../core/preview/index.js'; import { check } from './check.js'; import { openInBrowser } from './open.js'; +import * as telemetryHandler from './telemetry.js'; import { loadConfig } from '../core/config.js'; import { printHelp, formatErrorMessage, formatConfigErrorMessage } from '../core/messages.js'; import { createSafeError } from '../core/util.js'; @@ -27,7 +31,8 @@ type CLICommand = | 'build' | 'preview' | 'reload' - | 'check'; + | 'check' + | 'telemetry'; /** Display --help flag */ function printAstroHelp() { @@ -41,6 +46,7 @@ function printAstroHelp() { ['build', 'Build a pre-compiled production-ready site.'], ['preview', 'Preview your build locally before deploying.'], ['check', 'Check your project for errors.'], + ['telemetry', 'Enable/disable anonymous data collection.'], ['--version', 'Show the version number and exit.'], ['--help', 'Show this help message.'], ], @@ -67,6 +73,7 @@ async function printVersion() { function resolveCommand(flags: Arguments): CLICommand { const cmd = flags._[2] as string; if (cmd === 'add') return 'add'; + if (cmd === 'telemetry') return 'telemetry'; if (flags.version) return 'version'; else if (flags.help) return 'help'; @@ -103,12 +110,28 @@ export async function cli(args: string[]) { } else if (flags.silent) { logging.level = 'silent'; } + const telemetry = new AstroTelemetry({ version: process.env.PACKAGE_VERSION ?? '' }); + + if (cmd === 'telemetry') { + try { + const subcommand = flags._[3]?.toString(); + return await telemetryHandler.update(subcommand, { flags, telemetry }); + } catch (err) { + return throwAndExit(err); + } + } switch (cmd) { case 'add': { try { const packages = flags._.slice(3) as string[]; - return await add(packages, { cwd: root, flags, logging }); + telemetry.record( + event.eventCliSession({ + astroVersion: process.env.PACKAGE_VERSION ?? '', + cliCommand: 'add', + }) + ); + return await add(packages, { cwd: root, flags, logging, telemetry }); } catch (err) { return throwAndExit(err); } @@ -116,7 +139,13 @@ export async function cli(args: string[]) { case 'dev': { try { const config = await loadConfig({ cwd: root, flags, cmd }); - await devServer(config, { logging }); + telemetry.record( + event.eventCliSession( + { astroVersion: process.env.PACKAGE_VERSION ?? '', cliCommand: 'dev' }, + config + ) + ); + await devServer(config, { logging, telemetry }); return await new Promise(() => {}); // lives forever } catch (err) { return throwAndExit(err); @@ -126,7 +155,13 @@ export async function cli(args: string[]) { case 'build': { try { const config = await loadConfig({ cwd: root, flags, cmd }); - return await build(config, { logging }); + telemetry.record( + event.eventCliSession( + { astroVersion: process.env.PACKAGE_VERSION ?? '', cliCommand: 'build' }, + config + ) + ); + return await build(config, { logging, telemetry }); } catch (err) { return throwAndExit(err); } @@ -134,6 +169,12 @@ export async function cli(args: string[]) { case 'check': { const config = await loadConfig({ cwd: root, flags, cmd }); + telemetry.record( + event.eventCliSession( + { astroVersion: process.env.PACKAGE_VERSION ?? '', cliCommand: 'check' }, + config + ) + ); const ret = await check(config); return process.exit(ret); } @@ -141,7 +182,13 @@ export async function cli(args: string[]) { case 'preview': { try { const config = await loadConfig({ cwd: root, flags, cmd }); - const server = await preview(config, { logging }); + telemetry.record( + event.eventCliSession( + { astroVersion: process.env.PACKAGE_VERSION ?? '', cliCommand: 'preview' }, + config + ) + ); + const server = await preview(config, { logging, telemetry }); return await server.closed(); // keep alive until the server is closed } catch (err) { return throwAndExit(err); @@ -150,6 +197,12 @@ export async function cli(args: string[]) { case 'docs': { try { + await telemetry.record( + event.eventCliSession({ + astroVersion: process.env.PACKAGE_VERSION ?? '', + cliCommand: 'docs', + }) + ); return await openInBrowser('https://docs.astro.build/'); } catch (err) { return throwAndExit(err); diff --git a/packages/astro/src/cli/telemetry.ts b/packages/astro/src/cli/telemetry.ts new file mode 100644 index 000000000..147e405ab --- /dev/null +++ b/packages/astro/src/cli/telemetry.ts @@ -0,0 +1,47 @@ +/* eslint-disable no-console */ +import type yargs from 'yargs-parser'; +import type { AstroTelemetry } from '@astrojs/telemetry'; + +import prompts from 'prompts'; +import * as msg from '../core/messages.js'; + +export interface TelemetryOptions { + flags: yargs.Arguments; + telemetry: AstroTelemetry; +} + +export async function update(subcommand: string, { flags, telemetry }: TelemetryOptions) { + const isValid = ['enable', 'disable', 'reset'].includes(subcommand); + + if (flags.help || !isValid) { + msg.printHelp({ + commandName: 'astro telemetry', + usage: '<enable|disable|reset>', + commands: [ + ['enable', 'Enable anonymous data collection.'], + ['disable', 'Disable anonymous data collection.'], + ['reset', 'Reset anonymous data collection settings.'], + ], + }); + return; + } + + switch (subcommand) { + case 'enable': { + telemetry.setEnabled(true); + console.log(msg.telemetryEnabled()); + return; + } + case 'disable': { + telemetry.setEnabled(false); + console.log(msg.telemetryDisabled()); + return; + } + case 'reset': { + telemetry.clear(); + console.log(msg.telemetryReset()); + return; + } + } +} + diff --git a/packages/astro/src/core/add/index.ts b/packages/astro/src/core/add/index.ts index 5dd6b5caa..072de0bfb 100644 --- a/packages/astro/src/core/add/index.ts +++ b/packages/astro/src/core/add/index.ts @@ -1,4 +1,5 @@ import type yargs from 'yargs-parser'; +import type { AstroTelemetry } from '@astrojs/telemetry'; import path from 'path'; import { existsSync, promises as fs } from 'fs'; import { execa } from 'execa'; @@ -24,6 +25,7 @@ import { appendForwardSlash } from '../path.js'; export interface AddOptions { logging: LogOptions; flags: yargs.Arguments; + telemetry: AstroTelemetry; cwd?: string; } @@ -33,7 +35,7 @@ export interface IntegrationInfo { dependencies: [name: string, version: string][]; } -export default async function add(names: string[], { cwd, flags, logging }: AddOptions) { +export default async function add(names: string[], { cwd, flags, logging, telemetry }: AddOptions) { if (flags.help) { printHelp({ commandName: 'astro add', diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts index d0c0a026c..98278f72a 100644 --- a/packages/astro/src/core/build/index.ts +++ b/packages/astro/src/core/build/index.ts @@ -1,5 +1,6 @@ import type { AstroConfig, BuildConfig, ManifestData } from '../../@types/astro'; import type { LogOptions } from '../logger/core'; +import type { AstroTelemetry } from '@astrojs/telemetry'; import fs from 'fs'; import * as colors from 'kleur/colors'; @@ -33,13 +34,11 @@ import { fixViteErrorMessage } from '../errors.js'; export interface BuildOptions { mode?: string; logging: LogOptions; + telemetry: AstroTelemetry; } /** `astro build` */ -export default async function build( - config: AstroConfig, - options: BuildOptions = { logging: nodeLogOptions } -): Promise<void> { +export default async function build(config: AstroConfig, options: BuildOptions): Promise<void> { applyPolyfill(); const builder = new AstroBuilder(config, options); await builder.run(); diff --git a/packages/astro/src/core/dev/index.ts b/packages/astro/src/core/dev/index.ts index a1e7e528c..76051c623 100644 --- a/packages/astro/src/core/dev/index.ts +++ b/packages/astro/src/core/dev/index.ts @@ -1,4 +1,5 @@ import type { AddressInfo } from 'net'; +import type { AstroTelemetry } from '@astrojs/telemetry'; import { performance } from 'perf_hooks'; import * as vite from 'vite'; import type { AstroConfig } from '../../@types/astro'; @@ -17,6 +18,7 @@ import { apply as applyPolyfill } from '../polyfill.js'; export interface DevOptions { logging: LogOptions; + telemetry: AstroTelemetry; } export interface DevServer { @@ -25,12 +27,10 @@ export interface DevServer { } /** `astro dev` */ -export default async function dev( - config: AstroConfig, - options: DevOptions = { logging: nodeLogOptions } -): Promise<DevServer> { +export default async function dev(config: AstroConfig, options: DevOptions): Promise<DevServer> { const devStart = performance.now(); applyPolyfill(); + await options.telemetry.record([]); config = await runHookConfigSetup({ config, command: 'dev' }); const { host, port } = config.server; const viteConfig = await createVite( diff --git a/packages/astro/src/core/messages.ts b/packages/astro/src/core/messages.ts index fdec71396..38f9d78f5 100644 --- a/packages/astro/src/core/messages.ts +++ b/packages/astro/src/core/messages.ts @@ -1,7 +1,6 @@ -/** - * Dev server messages (organized here to prevent clutter) - */ - +import type { AddressInfo } from 'net'; +import type { AstroConfig } from '../@types/astro'; +import os from 'os'; import { bold, dim, @@ -15,10 +14,9 @@ import { black, bgRed, bgWhite, + bgCyan, } from 'kleur/colors'; -import os from 'os'; -import type { AddressInfo } from 'net'; -import type { AstroConfig } from '../@types/astro'; +import boxen from 'boxen'; import { collectErrorMetadata, cleanErrorStack } from './errors.js'; import { ZodError } from 'zod'; import { emoji, getLocalAddress, padMultilineString } from './util.js'; @@ -116,6 +114,37 @@ export function devStart({ return messages.map((msg) => ` ${msg}`).join('\n'); } +export function telemetryNotice() { + const headline = yellow(`Astro now collects ${bold('anonymous')} usage data.`); + const why = `This ${bold('optional program')} will help shape our roadmap.`; + const more = `For more info, visit ${underline('https://astro.build/telemetry')}`; + const box = boxen([headline, why, '', more].join('\n'), { + margin: 0, + padding: 1, + borderStyle: 'round', + borderColor: 'yellow', + }); + return box; +} + +export function telemetryEnabled() { + return `\n ${green('◉')} Anonymous telemetry is ${bgGreen( + black(' enabled ') + )}. Thank you for improving Astro!\n`; +} + +export function telemetryDisabled() { + return `\n ${yellow('◯')} Anonymous telemetry is ${bgYellow( + black(' disabled ') + )}. We won't share any usage data.\n`; +} + +export function telemetryReset() { + return `\n ${cyan('◆')} Anonymous telemetry has been ${bgCyan( + black(' reset ') + )}. You may be prompted again.\n`; +} + export function prerelease({ currentVersion }: { currentVersion: string }) { const tag = currentVersion.split('-').slice(1).join('-').replace(/\..*$/, ''); const badge = bgYellow(black(` ${tag} `)); @@ -227,7 +256,7 @@ export function printHelp({ for (const row of rows) { raw += `${opts.prefix}${bold(`${row[0]}`.padStart(opts.padding - opts.prefix.length))}`; if (split) raw += '\n '; - raw += dim(row[1]) + '\n'; + raw += ' ' + dim(row[1]) + '\n'; } return raw.slice(0, -1); // remove latest \n @@ -252,7 +281,7 @@ export function printHelp({ message.push( linebreak(), title('Commands'), - table(commands, { padding: 28, prefix: ' astro ' }) + table(commands, { padding: 28, prefix: ` ${commandName || 'astro'} ` }) ); } diff --git a/packages/astro/src/core/preview/index.ts b/packages/astro/src/core/preview/index.ts index e3303ec8e..4a952e380 100644 --- a/packages/astro/src/core/preview/index.ts +++ b/packages/astro/src/core/preview/index.ts @@ -1,6 +1,8 @@ import type { AstroConfig } from '../../@types/astro'; import type { LogOptions } from '../logger/core'; import type { AddressInfo } from 'net'; +import type { AstroTelemetry } from '@astrojs/telemetry'; + import http from 'http'; import sirv from 'sirv'; import { performance } from 'perf_hooks'; @@ -12,6 +14,7 @@ import { getResolvedHostForHttpServer } from './util.js'; interface PreviewOptions { logging: LogOptions; + telemetry: AstroTelemetry; } export interface PreviewServer { diff --git a/packages/astro/test/test-utils.js b/packages/astro/test/test-utils.js index aabb24ce5..267e4039f 100644 --- a/packages/astro/test/test-utils.js +++ b/packages/astro/test/test-utils.js @@ -86,10 +86,17 @@ export async function loadFixture(inlineConfig) { level: 'error', }; + /** @type {import('@astrojs/telemetry').AstroTelemetry} */ + const telemetry = { + record() { + return Promise.resolve(); + }, + }; + return { - build: (opts = {}) => build(config, { mode: 'development', logging, ...opts }), + build: (opts = {}) => build(config, { mode: 'development', logging, telemetry, ...opts }), startDevServer: async (opts = {}) => { - const devResult = await dev(config, { logging, ...opts }); + const devResult = await dev(config, { logging, telemetry, ...opts }); config.server.port = devResult.address.port; // update port return devResult; }, @@ -97,7 +104,7 @@ export async function loadFixture(inlineConfig) { fetch: (url, init) => fetch(`http://${'127.0.0.1'}:${config.server.port}${url.replace(/^\/?/, '/')}`, init), preview: async (opts = {}) => { - const previewServer = await preview(config, { logging, ...opts }); + const previewServer = await preview(config, { logging, telemetry, ...opts }); return previewServer; }, readFile: (filePath) => diff --git a/packages/telemetry/README.md b/packages/telemetry/README.md new file mode 100644 index 000000000..9c0999e1a --- /dev/null +++ b/packages/telemetry/README.md @@ -0,0 +1,9 @@ +# Astro Telemetry + +This package is used to collect anonymous telemetry data within the Astro CLI. Telemetry data does not contain any personal identifying information and can be disabled via: + +```shell +astro telemetry disable +``` + +See the [CLI documentation](https://docs.astro.build/en/reference/cli-reference/#astro-telemetry) for more options on configuration telemetry. diff --git a/packages/telemetry/events.d.ts b/packages/telemetry/events.d.ts new file mode 100644 index 000000000..e1bf09518 --- /dev/null +++ b/packages/telemetry/events.d.ts @@ -0,0 +1 @@ +export * from './dist/types/events'; diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json new file mode 100644 index 000000000..63b4c5edb --- /dev/null +++ b/packages/telemetry/package.json @@ -0,0 +1,47 @@ +{ + "name": "@astrojs/telemetry", + "version": "0.0.1", + "type": "module", + "types": "./dist/types/index.d.ts", + "author": "withastro", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/withastro/astro.git", + "directory": "packages/telemetry" + }, + "bugs": "https://github.com/withastro/astro/issues", + "homepage": "https://astro.build", + "exports": { + ".": "./dist/index.js", + "./events": "./dist/events/index.js", + "./package.json": "./package.json" + }, + "scripts": { + "build": "astro-scripts build \"src/**/*.ts\" && tsc", + "build:ci": "astro-scripts build \"src/**/*.ts\"", + "dev": "astro-scripts dev \"src/**/*.ts\"", + "test": "mocha --exit --timeout 20000 test/" + }, + "files": [ + "dist" + ], + "dependencies": { + "ci-info": "^3.3.0", + "debug": "^4.3.4", + "dlv": "^1.1.3", + "dset": "^3.1.1", + "escalade": "^3.1.1", + "is-docker": "^3.0.0", + "is-wsl": "^2.2.0", + "node-fetch": "^3.2.3" + }, + "devDependencies": { + "@types/dlv": "^1.1.2", + "@types/node": "^14.18.13", + "astro-scripts": "workspace:*" + }, + "engines": { + "node": "^14.15.0 || >=16.0.0" + } +} diff --git a/packages/telemetry/src/anonymous-meta.ts b/packages/telemetry/src/anonymous-meta.ts new file mode 100644 index 000000000..3c9e728b7 --- /dev/null +++ b/packages/telemetry/src/anonymous-meta.ts @@ -0,0 +1,48 @@ +import os from 'node:os'; +import isDocker from 'is-docker'; +import isWSL from 'is-wsl'; +import { isCI, name as ciName } from 'ci-info'; + +type AnonymousMeta = { + systemPlatform: NodeJS.Platform; + systemRelease: string; + systemArchitecture: string; + cpuCount: number; + cpuModel: string | null; + cpuSpeed: number | null; + memoryInMb: number; + isDocker: boolean; + isWSL: boolean; + isCI: boolean; + ciName: string | null; + astroVersion: string; +}; + +let meta: AnonymousMeta | undefined; + +export function getAnonymousMeta(astroVersion: string): AnonymousMeta { + if (meta) { + return meta; + } + + const cpus = os.cpus() || []; + meta = { + // Software information + systemPlatform: os.platform(), + systemRelease: os.release(), + systemArchitecture: os.arch(), + // Machine information + cpuCount: cpus.length, + cpuModel: cpus.length ? cpus[0].model : null, + cpuSpeed: cpus.length ? cpus[0].speed : null, + memoryInMb: Math.trunc(os.totalmem() / Math.pow(1024, 2)), + // Environment information + isDocker: isDocker(), + isWSL, + isCI, + ciName, + astroVersion, + }; + + return meta!; +} diff --git a/packages/telemetry/src/config.ts b/packages/telemetry/src/config.ts new file mode 100644 index 000000000..9d9bf21f9 --- /dev/null +++ b/packages/telemetry/src/config.ts @@ -0,0 +1,89 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; +import process from 'node:process'; +import { dset } from 'dset'; +import dget from 'dlv'; + +export interface ConfigOptions { + name: string; + defaults: Map<string, any>; +} + +// Adapted from https://github.com/sindresorhus/env-paths +function getConfigDir(name: string) { + const homedir = os.homedir(); + const macos = () => path.join(homedir, 'Library', 'Preferences', name); + const win = () => { + const { APPDATA = path.join(homedir, 'AppData', 'Roaming') } = process.env; + return path.join(APPDATA, name, 'Config'); + }; + const linux = () => { + const { XDG_CONFIG_HOME = path.join(homedir, '.config') } = process.env; + return path.join(XDG_CONFIG_HOME, name); + }; + switch (process.platform) { + case 'darwin': + return macos(); + case 'win32': + return win(); + default: + return linux(); + } +} + +export class Config { + private dir: string; + private file: string; + + constructor(private project: ConfigOptions) { + this.dir = getConfigDir(this.project.name); + this.file = path.join(this.dir, 'config.json'); + } + + private _store?: Record<string, any>; + private get store(): Record<string, any> { + if (this._store) return this._store; + this.ensureDir(); + if (fs.existsSync(this.file)) { + this._store = JSON.parse(fs.readFileSync(this.file).toString()); + } else { + const store = {}; + for (const [key, value] of this.project.defaults) { + dset(store, key, value); + } + this._store = store; + this.write(); + } + return this._store!; + } + private set store(value: Record<string, any>) { + this._store = value; + this.write(); + } + private ensureDir() { + fs.mkdirSync(this.dir, { recursive: true }); + } + write() { + fs.writeFileSync(this.file, JSON.stringify(this.store, null, '\t')); + } + clear(): void { + this.store = {}; + fs.rmSync(this.file, { recursive: true }); + } + delete(key: string): boolean { + dset(this.store, key, undefined); + this.write(); + return true; + } + get(key: string): any { + return dget(this.store, key); + } + has(key: string): boolean { + return typeof this.get(key) !== 'undefined'; + } + set(key: string, value: any): void { + dset(this.store, key, value); + this.write(); + } +} diff --git a/packages/telemetry/src/events/build.ts b/packages/telemetry/src/events/build.ts new file mode 100644 index 000000000..1d6b8b7fd --- /dev/null +++ b/packages/telemetry/src/events/build.ts @@ -0,0 +1,2 @@ +// See https://github.com/vercel/next.js/blob/canary/packages/next/telemetry/events/build.ts +export {}; diff --git a/packages/telemetry/src/events/index.ts b/packages/telemetry/src/events/index.ts new file mode 100644 index 000000000..20fc79a75 --- /dev/null +++ b/packages/telemetry/src/events/index.ts @@ -0,0 +1,2 @@ +export * from './session.js'; +export * from './build.js'; diff --git a/packages/telemetry/src/events/session.ts b/packages/telemetry/src/events/session.ts new file mode 100644 index 000000000..fce5976aa --- /dev/null +++ b/packages/telemetry/src/events/session.ts @@ -0,0 +1,96 @@ +import escalade from 'escalade/sync'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +const require = createRequire(import.meta.url); + +const EVENT_SESSION = 'ASTRO_CLI_SESSION_STARTED'; + +interface EventCliSession { + astroVersion: string; + cliCommand: string; +} + +interface ConfigInfo { + hasViteConfig: boolean; + hasBase: boolean; + viteKeys: string[]; + markdownPlugins: string[]; + adapter: string | null; + integrations: string[]; + experimentalFeatures: string[]; +} + +interface EventCliSessionInternal extends EventCliSession { + nodeVersion: string; + viteVersion: string; + config?: ConfigInfo; +} + +function getViteVersion() { + try { + const { version } = require('vite/package.json'); + return version; + } catch (e) {} + return undefined; +} + +function getExperimentalFeatures(astroConfig?: Record<string, any>): string[] | undefined { + if (!astroConfig) return undefined; + return Object.entries(astroConfig.experimental || []).reduce((acc, [key, value]) => { + if (value) { + acc.push(key); + } + return acc; + }, [] as string[]); +} + +const secondLevelViteKeys = new Set(["resolve", "css", "json", "server", "server.fs", "build", "preview", "optimizeDeps", "ssr", "worker"]); +function viteConfigKeys(obj: Record<string, any> | undefined, parentKey: string): string[] { + if(!obj) { + return []; + } + + return Object.entries(obj).map(([key, value]) => { + if(typeof value === 'object' && !Array.isArray(value)) { + const localKey = parentKey ? parentKey + '.' + key : key; + if(secondLevelViteKeys.has(localKey)) { + let keys = viteConfigKeys(value, localKey).map(subkey => key + '.' + subkey); + keys.unshift(key); + return keys; + } + } + + return key; + }).flat(1); +} + +export function eventCliSession( + event: EventCliSession, + astroConfig?: Record<string, any> +): { eventName: string; payload: EventCliSessionInternal }[] { + const payload: EventCliSessionInternal = { + cliCommand: event.cliCommand, + // Versions + astroVersion: event.astroVersion, + viteVersion: getViteVersion(), + nodeVersion: process.version.replace(/^v?/, ''), + // Config Values + config: astroConfig + ? { + hasViteConfig: Object.keys(astroConfig?.vite).length > 0, + markdownPlugins: + [ + astroConfig?.markdown?.remarkPlugins ?? [], + astroConfig?.markdown?.rehypePlugins ?? [], + ].flat(1), + hasBase: astroConfig?.base !== '/', + viteKeys: viteConfigKeys(astroConfig?.vite, ''), + adapter: astroConfig?.adapter?.name ?? null, + integrations: astroConfig?.integrations?.map((i: any) => i.name) ?? [], + experimentalFeatures: getExperimentalFeatures(astroConfig) ?? [], + } + : undefined, + }; + return [{ eventName: EVENT_SESSION, payload }]; +} diff --git a/packages/telemetry/src/index.ts b/packages/telemetry/src/index.ts new file mode 100644 index 000000000..ef7157dfc --- /dev/null +++ b/packages/telemetry/src/index.ts @@ -0,0 +1,170 @@ +import type { BinaryLike } from 'node:crypto'; +import { createHash, randomBytes } from 'node:crypto'; + +import { isCI } from 'ci-info'; +import debug from 'debug'; + +import * as KEY from './keys.js'; +import { post } from './post.js'; +import { getAnonymousMeta } from './anonymous-meta.js'; +import { getRawProjectId } from './project-id.js'; +import { Config } from './config.js'; + +export interface AstroTelemetryOptions { + version: string; +} + +export type TelemetryEvent = { eventName: string; payload: Record<string, any> }; + +interface EventContext { + anonymousId: string; + projectId: string; + sessionId: string; +} + +export class AstroTelemetry { + private rawProjectId = getRawProjectId(); + private sessionId = randomBytes(32).toString('hex'); + private config = new Config({ + name: 'astro', + // Use getter to defer generation of defaults unless needed + get defaults() { + return new Map<string, any>([ + [KEY.TELEMETRY_ENABLED, true], + [KEY.TELEMETRY_SALT, randomBytes(16).toString('hex')], + [KEY.TELEMETRY_ID, randomBytes(32).toString('hex')], + ]); + }, + }); + private debug = debug('astro:telemetry'); + + private get astroVersion() { + return this.opts.version; + } + private get ASTRO_TELEMETRY_DISABLED() { + return process.env.ASTRO_TELEMETRY_DISABLED; + } + private get TELEMETRY_DISABLED() { + return process.env.TELEMETRY_DISABLED; + } + + constructor(private opts: AstroTelemetryOptions) { + // When the process exits, flush any queued promises + process.on('SIGINT', () => this.flush()); + } + + // Util to get value from config or set it if missing + private getWithFallback<T>(key: string, value: T): T { + const val = this.config.get(key); + if (val) { + return val; + } + this.config.set(key, value); + return value; + } + + private get salt(): string { + return this.getWithFallback(KEY.TELEMETRY_SALT, randomBytes(16).toString('hex')); + } + private get enabled(): boolean { + return this.getWithFallback(KEY.TELEMETRY_ENABLED, true); + } + private get anonymousId(): string { + return this.getWithFallback(KEY.TELEMETRY_ID, randomBytes(32).toString('hex')); + } + private get notifyDate(): string { + return this.getWithFallback(KEY.TELEMETRY_NOTIFY_DATE, ''); + } + + // Create a ONE-WAY hash so there is no way for Astro to decode the value later. + private oneWayHash(payload: BinaryLike): string { + const hash = createHash('sha256'); + // Always prepend the payload value with salt! This ensures the hash is one-way. + hash.update(this.salt); + hash.update(payload); + return hash.digest('hex'); + } + + // Instead of sending `rawProjectId`, we only ever reference a hashed value *derived* + // from `rawProjectId`. This ensures that `projectId` is ALWAYS anonymous and can't + // be reversed from the hashed value. + private get projectId(): string { + return this.oneWayHash(this.rawProjectId); + } + + private get isDisabled(): boolean { + if (Boolean(this.ASTRO_TELEMETRY_DISABLED || this.TELEMETRY_DISABLED)) { + return true; + } + return this.enabled === false; + } + + setEnabled(value: boolean) { + this.config.set(KEY.TELEMETRY_ENABLED, value); + } + + clear() { + return this.config.clear(); + } + + private queue: Promise<any>[] = []; + + // Wait for any in-flight promises to resolve + private async flush() { + await Promise.all(this.queue); + } + + async notify(callback: () => Promise<boolean>) { + if (this.isDisabled || isCI) { + return; + } + // The end-user has already been notified about our telemetry integration! + // Don't bother them about it again. + // In the event of significant changes, we should invalidate old dates. + if (this.notifyDate) { + return; + } + const enabled = await callback(); + this.config.set(KEY.TELEMETRY_NOTIFY_DATE, Date.now().toString()); + this.config.set(KEY.TELEMETRY_ENABLED, enabled); + } + + async record(event: TelemetryEvent | TelemetryEvent[] = []) { + const events: TelemetryEvent[] = Array.isArray(event) ? event : [event]; + if (events.length < 1) { + return Promise.resolve(); + } + + if (this.debug.enabled) { + // Print to standard error to simplify selecting the output + events.forEach(({ eventName, payload }) => + this.debug(JSON.stringify({ eventName, payload }, null, 2)) + ); + // Do not send the telemetry data if debugging. Users may use this feature + // to preview what data would be sent. + return Promise.resolve(); + } + + // Skip recording telemetry if the feature is disabled + if (this.isDisabled) { + return Promise.resolve(); + } + + const context: EventContext = { + anonymousId: this.anonymousId, + projectId: this.projectId, + sessionId: this.sessionId, + }; + const meta = getAnonymousMeta(this.astroVersion); + + const req = post({ + context, + meta, + events, + }).then(() => { + this.queue = this.queue.filter((r) => r !== req); + }); + this.queue.push(req); + return req; + } +} diff --git a/packages/telemetry/src/keys.ts b/packages/telemetry/src/keys.ts new file mode 100644 index 000000000..f1c9e2ad2 --- /dev/null +++ b/packages/telemetry/src/keys.ts @@ -0,0 +1,16 @@ +// This is the key that stores whether or not telemetry is enabled or disabled. +export const TELEMETRY_ENABLED = 'telemetry.enabled'; + +// This is the key that specifies when the user was informed about anonymous +// telemetry collection. +export const TELEMETRY_NOTIFY_DATE = 'telemetry.notifiedAt'; + +// This is a quasi-persistent identifier used to dedupe recurring events. It's +// generated from random data and completely anonymous. +export const TELEMETRY_ID = `telemetry.anonymousId`; + +// This is the cryptographic salt that is included within every hashed value. +// This salt value is never sent to us, ensuring privacy and the one-way nature +// of the hash (prevents dictionary lookups of pre-computed hashes). +// See the `oneWayHash` function. +export const TELEMETRY_SALT = `telemetry.salt`; diff --git a/packages/telemetry/src/post.ts b/packages/telemetry/src/post.ts new file mode 100644 index 000000000..ae1626a40 --- /dev/null +++ b/packages/telemetry/src/post.ts @@ -0,0 +1,13 @@ +import fetch from 'node-fetch'; +const ASTRO_TELEMETRY_ENDPOINT = `https://telemetry.astro.build/api/v1/record`; +const noop = () => {}; + +export function post(body: Record<string, any>) { + return fetch(ASTRO_TELEMETRY_ENDPOINT, { + method: 'POST', + body: JSON.stringify(body), + headers: { 'content-type': 'application/json' }, + }) + .catch(noop) + .then(noop, noop); +} diff --git a/packages/telemetry/src/project-id.ts b/packages/telemetry/src/project-id.ts new file mode 100644 index 000000000..655a72fc6 --- /dev/null +++ b/packages/telemetry/src/project-id.ts @@ -0,0 +1,27 @@ +import { execSync } from 'child_process'; + +// Why does Astro need a project ID? Why is it looking at my git remote? +// --- +// Astro's telemetry is and always will be completely anonymous. +// Differentiating unique projects helps us track feature usage accurately. +// +// We **never** read your actual git remote! The value is hashed one-way +// with random salt data, making it impossible for us to reverse or try to +// guess the remote by re-computing hashes. + +function getProjectIdFromGit() { + try { + const originBuffer = execSync(`git config --local --get remote.origin.url`, { + timeout: 1000, + stdio: `pipe`, + }); + + return String(originBuffer).trim(); + } catch (_) { + return null; + } +} + +export function getRawProjectId(): string { + return getProjectIdFromGit() ?? process.env.REPOSITORY_URL ?? process.cwd(); +} diff --git a/packages/telemetry/test/session-event.test.js b/packages/telemetry/test/session-event.test.js new file mode 100644 index 000000000..5a23bade8 --- /dev/null +++ b/packages/telemetry/test/session-event.test.js @@ -0,0 +1,181 @@ +import { expect } from 'chai'; +import * as events from '../dist/events/index.js'; +import { resolveConfig } from '../../astro/dist/core/config.js'; + +async function mockConfig(userConfig) { + return await resolveConfig(userConfig, import.meta.url, {}, 'dev'); +} + +describe('Session event', () => { + it('top-level keys are captured', async () => { + const config = await mockConfig({ + vite: { + css: { modules: [] }, + base: 'a', + mode: 'b', + define: { + a: 'b', + }, + publicDir: 'some/dir', + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['css', 'css.modules', 'base', 'mode', 'define', 'publicDir']); + }) + + it('vite.resolve keys are captured', async () => { + const config = await mockConfig({ + vite: { + resolve: { + alias: { + a: 'b' + }, + dedupe: ['one', 'two'] + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['resolve', 'resolve.alias', 'resolve.dedupe']); + }); + + it('vite.css keys are captured', async () => { + const config = await mockConfig({ + vite: { + resolve: { + dedupe: ['one', 'two'] + }, + css: { + modules: [], + postcss: {} + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['resolve', 'resolve.dedupe', 'css', 'css.modules', 'css.postcss']); + }); + + it('vite.server keys are captured', async () => { + const config = await mockConfig({ + vite: { + server: { + host: 'example.com', + open: true, + fs: { + strict: true, + allow: ['a', 'b'] + } + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['server', 'server.host', 'server.open', 'server.fs', 'server.fs.strict', 'server.fs.allow']); + }); + + it('vite.build keys are captured', async () => { + const config = await mockConfig({ + vite: { + build: { + target: 'one', + outDir: 'some/dir', + cssTarget: { + one: 'two' + } + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['build', 'build.target', 'build.outDir', 'build.cssTarget']); + }); + + + it('vite.preview keys are captured', async () => { + const config = await mockConfig({ + vite: { + preview: { + host: 'example.com', + port: 8080, + another: { + a: 'b' + } + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['preview', 'preview.host', 'preview.port', 'preview.another']); + }); + + it('vite.optimizeDeps keys are captured', async () => { + const config = await mockConfig({ + vite: { + optimizeDeps: { + entries: ['one', 'two'], + exclude: ['secret', 'name'] + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['optimizeDeps', 'optimizeDeps.entries', 'optimizeDeps.exclude']); + }); + + it('vite.ssr keys are captured', async () => { + const config = await mockConfig({ + vite: { + ssr: { + external: ['a'], + target: { one: 'two' } + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['ssr', 'ssr.external', 'ssr.target']); + }); + + it('vite.worker keys are captured', async () => { + const config = await mockConfig({ + vite: { + worker: { + format: { a: 'b' }, + plugins: ['a', 'b'] + } + } + }); + + const [{ payload }] = events.eventCliSession({ + cliCommand: 'dev', + astroVersion: '0.0.0' + }, config); + expect(payload.config.viteKeys).is.deep.equal(['worker', 'worker.format', 'worker.plugins']); + }); +}); diff --git a/packages/telemetry/tsconfig.json b/packages/telemetry/tsconfig.json new file mode 100644 index 000000000..8ee4c8711 --- /dev/null +++ b/packages/telemetry/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src"], + "compilerOptions": { + "allowJs": true, + "target": "ES2020", + "module": "ES2020", + "outDir": "./dist", + "declarationDir": "./dist/types" + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 47696f246..1f8157973 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -461,6 +461,7 @@ importers: '@astrojs/language-server': ^0.13.4 '@astrojs/markdown-remark': ^0.9.2 '@astrojs/prism': 0.4.1 + '@astrojs/telemetry': ^0.0.1 '@astrojs/webapi': ^0.11.1 '@babel/core': ^7.17.9 '@babel/generator': ^7.17.9 @@ -546,6 +547,7 @@ importers: '@astrojs/language-server': 0.13.4 '@astrojs/markdown-remark': link:../markdown/remark '@astrojs/prism': link:../astro-prism + '@astrojs/telemetry': link:../telemetry '@astrojs/webapi': link:../webapi '@babel/core': 7.17.9 '@babel/generator': 7.17.9 @@ -1521,6 +1523,33 @@ importers: '@types/unist': 2.0.6 astro-scripts: link:../../../scripts + packages/telemetry: + specifiers: + '@types/dlv': ^1.1.2 + '@types/node': ^14.18.13 + astro-scripts: workspace:* + ci-info: ^3.3.0 + debug: ^4.3.4 + dlv: ^1.1.3 + dset: ^3.1.1 + escalade: ^3.1.1 + is-docker: ^3.0.0 + is-wsl: ^2.2.0 + node-fetch: ^3.2.3 + dependencies: + ci-info: 3.3.0 + debug: 4.3.4 + dlv: 1.1.3 + dset: 3.1.1 + escalade: 3.1.1 + is-docker: 3.0.0 + is-wsl: 2.2.0 + node-fetch: 3.2.3 + devDependencies: + '@types/dlv': 1.1.2 + '@types/node': 14.18.13 + astro-scripts: link:../../scripts + packages/webapi: specifiers: '@rollup/plugin-alias': ^3.1.9 @@ -3881,6 +3910,10 @@ packages: resolution: {integrity: sha512-uw8eYMIReOwstQ0QKF0sICefSy8cNO/v7gOTiIy9SbwuHyEecJUm7qlgueOO5S1udZ5I/irVydHVwMchgzbKTg==} dev: true + /@types/dlv/1.1.2: + resolution: {integrity: sha512-OyiZ3jEKu7RtGO1yp9oOdK0cTwZ/10oE9PDJ6fyN3r9T5wkyOcvr6awdugjYdqF6KVO5eUvt7jx7rk2Eylufow==} + dev: true + /@types/estree-jsx/0.0.1: resolution: {integrity: sha512-gcLAYiMfQklDCPjQegGn0TBAn9it05ISEsEhlKQUddIk7o2XDokOcTN7HBO8tznM0D9dGezvHEfRZBfZf6me0A==} dependencies: @@ -5293,7 +5326,6 @@ packages: /data-uri-to-buffer/4.0.0: resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==} engines: {node: '>= 12'} - dev: true /dataloader/1.4.0: resolution: {integrity: sha512-68s5jYdlvasItOJnCuI2Q9s4q98g0pCyL3HrcKJu8KNugUl8ahgmZYg38ysLTgQjjXX3H8CJLkAvWrclWfcalw==} @@ -5528,6 +5560,11 @@ packages: engines: {node: '>=10'} dev: true + /dset/3.1.1: + resolution: {integrity: sha512-hYf+jZNNqJBD2GiMYb+5mqOIX4R4RRHXU3qWMWYN+rqcR2/YpRL2bUHr8C8fU+5DNvqYjJ8YvMGSLuVPWU1cNg==} + engines: {node: '>=4'} + dev: false + /duplexer/0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: true @@ -6137,7 +6174,6 @@ packages: dependencies: node-domexception: 1.0.0 web-streams-polyfill: 3.2.1 - dev: true /file-entry-cache/6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} @@ -6215,7 +6251,6 @@ packages: engines: {node: '>=12.20.0'} dependencies: fetch-blob: 3.1.5 - dev: true /fraction.js/4.2.0: resolution: {integrity: sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==} @@ -6872,7 +6907,12 @@ packages: resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} engines: {node: '>=8'} hasBin: true - dev: true + + /is-docker/3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + dev: false /is-extendable/0.1.1: resolution: {integrity: sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=} @@ -7061,7 +7101,6 @@ packages: engines: {node: '>=8'} dependencies: is-docker: 2.2.1 - dev: true /isarray/0.0.1: resolution: {integrity: sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=} @@ -8057,7 +8096,6 @@ packages: /node-domexception/1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} - dev: true /node-fetch/2.6.7: resolution: {integrity: sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==} @@ -8077,7 +8115,6 @@ packages: data-uri-to-buffer: 4.0.0 fetch-blob: 3.1.5 formdata-polyfill: 4.0.10 - dev: true /node-releases/2.0.3: resolution: {integrity: sha512-maHFz6OLqYxz+VQyCAtA3PTX4UP/53pa05fyDNc9CwjvJ0yEh6+xBwKsgCxMNhS8taUKBFYxfuiaD9U/55iFaw==} @@ -10507,7 +10544,6 @@ packages: /web-streams-polyfill/3.2.1: resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} engines: {node: '>= 8'} - dev: true /webidl-conversions/3.0.1: resolution: {integrity: sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=} diff --git a/scripts/memory/index.js b/scripts/memory/index.js index da55acb71..30cc1c0f2 100644 --- a/scripts/memory/index.js +++ b/scripts/memory/index.js @@ -18,7 +18,12 @@ let config = await loadConfig({ cwd: fileURLToPath(projDir), }); -const server = await dev(config, { logging: { level: 'error' } }); +const telemetry = { + record() { + return Promise.resolve(); + }, +}; +const server = await dev(config, { logging: { level: 'error' }, telemetry }); // Prime the server so initial memory is created await fetch(`http://localhost:3000/page-0`); diff --git a/scripts/smoke/index.js b/scripts/smoke/index.js index 354147cd6..d231b5e5d 100644 --- a/scripts/smoke/index.js +++ b/scripts/smoke/index.js @@ -47,6 +47,7 @@ async function run() { try { await execa('pnpm', ['install', '--ignore-scripts', '--frozen-lockfile=false', isExternal ? '--shamefully-hoist' : ''].filter(x => x), { cwd: fileURLToPath(directory), stdio: 'inherit' }); + await execa('pnpm', ['astro', 'telemetry', 'disable']); await execa('pnpm', ['run', 'build'], { cwd: fileURLToPath(directory), stdio: 'inherit' }); } catch (err) { console.log(err); |