import * as fs from 'node:fs'; import { pathToFileURL } from 'node:url'; import type { MatrixActor } from '@open-matrix/core/core/MatrixActor.js'; import { buildMountedInstanceBootstrapProps } from '@open-matrix/core/runtime/InstanceBootstrapContext.js'; import { buildMatrixHttpAssetMount, MatrixHttpAssetEndpointActor, } from '@open-matrix/system-gateway-http'; import { HostAuthStateStore, HostCredentialStore } from '@open-matrix/system-auth'; import { loadBootstrapJsonObject, resolveBootstrapRefPath, loadBootstrapJsonValue, } from '@open-matrix/core/runtime/InstanceBootstrapRefs.js'; import { MatrixRuntime } from '@open-matrix/core/runtime/MatrixRuntime.js'; import { loadMatrixServiceManifest } from '../utils/service-manifest.js'; import { loadPackageEnvironment, loadPackageEnvironmentFromPath, type IMatrixPackageEnvironment, } from '../utils/package-environment.js'; import { deregisterRunnerBindings, deregisterRunnerRuntime, heartbeatRunnerRuntime, type IRunnerWebappRouteMetadata, registerRunnerBindings, type IRunnerBindingDeregistration, type IRunnerBindingRegistration, registerRunnerRuntime, renewRunnerBindings, resolveRunnerRuntimeId, type IRunnerRuntimeDeregistration, type IRunnerRuntimeRegistration, } from '../utils/runner-control-plane.js'; import { defaultRunnerRuntimeMount, mountRunnerRuntimeControl, type RunnerRuntimeLifecycleState, } from '../utils/runner-runtime-control.js'; import { resolveRunnerTransportPlan, type IResolvedRunnerTransportPlan } from '../utils/runner-transport.js'; import { createRunnerRuntimeHandle, formatRunnerRuntimeClosed, type IRunnerRuntimeClosed, type IRunnerRuntimeHandle, } from '../utils/runner-runtime.js'; import { loadMatrixWebappManifest, type ILoadedMatrixWebappManifest, } from '../utils/webapp-manifest.js'; import { installRunnerPlacementPolicy } from '../utils/runner-placement-policy.js'; export interface IServiceCommandOptions { readonly check?: boolean; readonly json?: boolean; readonly env?: string; readonly envFile?: string; readonly overrides?: string; readonly entry?: string; readonly export?: string; readonly root?: string; readonly mount?: string; readonly config?: string; readonly secrets?: string; } export interface IServiceCommandResult { readonly packageName: string; readonly packageDir: string; readonly manifestPath: string; readonly entryPath: string; readonly exportName: string; readonly checked: boolean; readonly environmentName?: string; readonly environmentPath?: string; readonly runnerTransport?: IResolvedRunnerTransportPlan; readonly overrides: Record; readonly root?: string; readonly mount?: string; readonly configRef?: string; readonly secretsRef?: string; readonly instance: Record; readonly serviceInstance: Record; readonly deploymentInstance: Record; readonly runtimeRegistration?: IRunnerRuntimeRegistration; readonly bindingRegistration?: IRunnerBindingRegistration; readonly runtimeDeregistration?: IRunnerRuntimeDeregistration; readonly bindingDeregistration?: IRunnerBindingDeregistration; readonly result?: unknown; } export interface IStartedServiceCommand { readonly result: IServiceCommandResult; readonly shutdownRequested: Promise; readonly transportClosed?: Promise; shutdown(): Promise; } type BootstrapResult = { runtime?: { shutdown?: () => Promise | void; }; } & Record; interface IStandaloneRunnerServices { readonly runtime: MatrixRuntime; readonly environment?: IMatrixPackageEnvironment; readonly runnerTransport?: IResolvedRunnerTransportPlan; } export async function serviceCommand( packageDir: string, options: IServiceCommandOptions = {}, ): Promise { const started = await startServiceCommand(packageDir, options); if (options.check) { return await started.shutdown(); } if (options.json) { console.log(JSON.stringify(started.result, null, 2)); } else { console.log(`[mx service] started ${started.result.packageName} via ${started.result.exportName}`); console.log(`[mx service] manifest: ${started.result.manifestPath}`); console.log(`[mx service] entry: ${started.result.entryPath}`); } let finalResult = started.result; const shutdownRequested = started.transportClosed ? Promise.race([ started.shutdownRequested, started.transportClosed.then((closed) => { throw new Error(formatRunnerRuntimeClosed(closed)); }), ]) : started.shutdownRequested; await waitForShutdownSignal(async () => { finalResult = await started.shutdown(); }, shutdownRequested); return finalResult; } export async function startServiceCommand( packageDir: string, options: IServiceCommandOptions = {}, ): Promise { const loaded = loadMatrixServiceManifest(packageDir); const webapp = loadMatrixWebappManifest(loaded.packageDir); const envFile = options.envFile?.trim(); const selectedEnvironment = envFile ? loadPackageEnvironmentFromPath(packageDir, envFile, options.env) : options.env?.trim() ? loadPackageEnvironment(packageDir, options.env) : undefined; const entryPath = options.entry ? loadBootstrapEntryPath(loaded.packageDir, options.entry, 'service entry override') : loaded.entryPath; const exportName = options.export?.trim() || loaded.exportName; const root = options.root?.trim() || selectedEnvironment?.environment.runtime.root || loaded.root; const runnerTransport = selectedEnvironment ? resolveRunnerTransportPlan(loaded.packageDir, selectedEnvironment, root) : undefined; const mount = options.mount?.trim() || loaded.mount; const configRef = options.config?.trim() || loaded.configRef; const config = configRef ? loadBootstrapJsonObject(loaded.packageDir, configRef, 'service config') : {}; const manifestOverrides = deepMerge(config, loaded.overrides); const cliOverrides = parseOverrides(options.overrides); const effectiveOverrides = deepMerge(manifestOverrides, cliOverrides); const secretsRef = options.secrets?.trim() || undefined; const secrets = secretsRef ? loadBootstrapJsonValue(loaded.packageDir, secretsRef, 'service secrets') : undefined; const mountedInstanceBootstrap = buildMountedInstanceBootstrapProps({ ...loaded.serviceInstance, ...(mount ? { mount } : {}), ...(configRef ? { configRef } : {}), ...(Array.isArray(secrets) ? { secretRefs: structuredClone(secrets) } : loaded.serviceInstance.secretRefs ? { secretRefs: structuredClone(loaded.serviceInstance.secretRefs) } : {}), }, effectiveOverrides); const { instance, serviceInstance, deploymentInstance } = mountedInstanceBootstrap; const bootstrapContext = { packageName: loaded.packageName, packageDir: loaded.packageDir, manifestPath: loaded.manifestPath, entryPath, exportName, ...(root ? { root } : {}), ...(mount ? { mount } : {}), ...(configRef ? { configRef } : {}), ...(secretsRef ? { secretsRef } : {}), ...(secrets !== undefined ? { secrets } : {}), ...(loaded.transport ? { transport: loaded.transport } : {}), ...(loaded.auth ? { auth: loaded.auth } : {}), ...(loaded.http ? { http: loaded.http } : {}), ...(selectedEnvironment ? { environmentName: selectedEnvironment.envName, environmentPath: selectedEnvironment.environmentPath, environment: selectedEnvironment.environment, } : {}), ...(runnerTransport ? { runnerTransport } : {}), ...mountedInstanceBootstrap, }; const runnerRuntimeHandle = await createRunnerRuntimeHandle(root, runnerTransport); const runnerRuntime = runnerRuntimeHandle.runtime; registerHostHomeService(runnerRuntime, selectedEnvironment?.environment.host?.matrixDir); const runtimeId = resolveRunnerRuntimeId(loaded, selectedEnvironment); const runtimeMount = selectedEnvironment?.environment.runtime.runtimeMount ?? process.env.MATRIX_RUNTIME_MOUNT ?? defaultRunnerRuntimeMount(runtimeId); // Slice 1b (control-actor-nav): control authority is the canonical // per-runtime mount, not the legacy `.control` sibling. const controlMount = selectedEnvironment?.environment.runtime.controlMount ?? process.env.MATRIX_RUNTIME_CONTROL_MOUNT ?? runtimeMount; let lifecycleState: RunnerRuntimeLifecycleState = 'starting'; let requestRuntimeShutdown: (reason: string) => void = () => undefined; const shutdownRequested = new Promise((resolve) => { requestRuntimeShutdown = resolve; }); const runnerServices: IStandaloneRunnerServices = { runtime: runnerRuntime, ...(selectedEnvironment ? { environment: selectedEnvironment.environment } : {}), ...(runnerTransport ? { runnerTransport } : {}), }; installRunnerPlacementPolicy({ runtime: runnerRuntime, manifest: loaded, environment: selectedEnvironment, runtimeId, authorityRoot: root, }); const moduleUrl = `${pathToFileURL(entryPath).href}?mx_service=${Date.now()}-${Math.random().toString(16).slice(2)}`; const mod = await import(moduleUrl) as Record; const bootstrap = mod[exportName]; if (typeof bootstrap !== 'function') { throw new Error(`Package runtime factory export "${exportName}" was not found in ${entryPath}`); } const bootstrapResult = await (bootstrap as (...args: unknown[]) => Promise | unknown)( effectiveOverrides, runnerServices, undefined, bootstrapContext, ); const activeRuntime = resolveBootstrapRuntime(bootstrapResult) ?? runnerRuntime; const webappAssetMount = webapp ? await mountMatrixHttpAssetEndpoint(activeRuntime, webapp, runtimeId) : undefined; const mounted = await mountRunnerRuntimeControl({ runtime: activeRuntime, runtimeId, runtimeMount, controlMount, startedAt: new Date().toISOString(), getState: () => lifecycleState, requestShutdown: (reason) => requestRuntimeShutdown(reason), loadedPackage: { packageRef: loaded.packageName, manifest: { packageRef: loaded.packageName, ...(loaded.mount ? { mount: loaded.mount } : {}), packageDir: loaded.packageDir, }, }, }); const mountedControlMount = mounted.controlMount; lifecycleState = 'ready'; const runtimeRegistration = await registerRunnerRuntime( activeRuntime, loaded, selectedEnvironment, runnerTransport, mountedInstanceBootstrap.serviceInstance.mount, runtimeMount, mountedControlMount, webapp && webappAssetMount ? buildRunnerWebappRouteMetadata(webapp, webappAssetMount) : undefined, ); const bindingRegistration = await registerRunnerBindings( activeRuntime, loaded, selectedEnvironment, runtimeRegistration.runtimeId, mountedInstanceBootstrap.serviceInstance.mount, ); lifecycleState = 'live'; const serializedInstance = sanitizeForJson(instance) as Record; let runtimeDeregistration: IRunnerRuntimeDeregistration | undefined; let bindingDeregistration: IRunnerBindingDeregistration | undefined; // Heartbeat tick + self-healing reclaim. // // registry.claim.renew (inside heartbeatRunnerRuntime / renewRunnerBindings) // returns the count of claims the server-side registry actually refreshed. // 0 means "the registry has no record of your claim" — typically because // the registry actor's in-memory _claims map was wiped when the SYSTEM // runtime restarted. Without reclaim the original claim's TTL (45s) // expires, the runtime vanishes from registry.query, the gateway can't // route to it, and apps appear "not registered" until manual intervention. // // We re-issue exactly what we issued at registration time. Server-side ops // are idempotent (onRegistryClaim preserves registeredAt/claimedAt for // existing entries; bindings.register is keyed by provider+match), so // reclaiming against a healthy registry is harmless. // // Log discipline: log STATE TRANSITIONS, not every tick. First failure // logs; subsequent consecutive failures at the same attempt count stay // silent; every 10th attempt logs that we're still trying; recovery from // a degraded state logs the duration. Healthy ticks log nothing. let runtimeReclaimAttempts = 0; let bindingReclaimAttempts = 0; const logIfTransitionOrMilestone = ( attempts: number, onFirst: () => void, onMilestone: () => void, ): void => { if (attempts === 1) { onFirst(); } else if (attempts % 10 === 0) { onMilestone(); } }; const runtimePresenceTimer = options.check === true ? undefined : setInterval(() => { void (async () => { try { const [heartbeat, renewal] = await Promise.all([ heartbeatRunnerRuntime(activeRuntime, runtimeRegistration), renewRunnerBindings(activeRuntime, bindingRegistration), ]); if (heartbeat && !heartbeat.known) { runtimeReclaimAttempts += 1; logIfTransitionOrMilestone( runtimeReclaimAttempts, () => console.error( `[mx service] registry forgot runtime claim for ${heartbeat.runtimeId} (renewed=0); reclaiming`, ), () => console.error( `[mx service] still reclaiming runtime ${heartbeat.runtimeId} after ${runtimeReclaimAttempts} attempts`, ), ); try { const reclaimed = await runtimeRegistration.reclaim(); if (runtimeReclaimAttempts > 0) { console.error( `[mx service] runtime reclaim recovered for ${heartbeat.runtimeId} after ${runtimeReclaimAttempts} attempt(s): ${reclaimed.inventoryClaimCount} inventory claims re-issued`, ); } runtimeReclaimAttempts = 0; } catch (reclaimErr) { logIfTransitionOrMilestone( runtimeReclaimAttempts, () => console.error( `[mx service] runtime reclaim failed for ${heartbeat.runtimeId}: ${reclaimErr instanceof Error ? reclaimErr.message : String(reclaimErr)} (will retry next tick)`, ), () => console.error( `[mx service] runtime reclaim still failing for ${heartbeat.runtimeId} after ${runtimeReclaimAttempts} attempts: ${reclaimErr instanceof Error ? reclaimErr.message : String(reclaimErr)}`, ), ); } } else if (heartbeat && runtimeReclaimAttempts > 0) { // Registry started answering renew=N>0 without us reclaiming // first — extremely rare (the registry would need to repopulate // claims from somewhere) but log the recovery so operators see // it. Reset the counter. console.error( `[mx service] runtime registry recovered without explicit reclaim for ${heartbeat.runtimeId} after ${runtimeReclaimAttempts} attempt(s)`, ); runtimeReclaimAttempts = 0; } if (renewal && renewal.refreshed === 0 && (bindingRegistration?.bindings.length ?? 0) > 0) { bindingReclaimAttempts += 1; logIfTransitionOrMilestone( bindingReclaimAttempts, () => console.error( `[mx service] registry forgot binding claims for ${renewal.runtimeId} (refreshed=0); reclaiming`, ), () => console.error( `[mx service] still reclaiming bindings for ${renewal.runtimeId} after ${bindingReclaimAttempts} attempts`, ), ); try { const reclaimed = await bindingRegistration?.reclaim(); if (reclaimed && bindingReclaimAttempts > 0) { console.error( `[mx service] binding reclaim recovered for ${renewal.runtimeId} after ${bindingReclaimAttempts} attempt(s): ${reclaimed.registryClaimCount} registry claims + ${reclaimed.bindingsRegisteredCount} binding forwards re-issued`, ); } bindingReclaimAttempts = 0; } catch (reclaimErr) { logIfTransitionOrMilestone( bindingReclaimAttempts, () => console.error( `[mx service] binding reclaim failed for ${renewal.runtimeId}: ${reclaimErr instanceof Error ? reclaimErr.message : String(reclaimErr)} (will retry next tick)`, ), () => console.error( `[mx service] binding reclaim still failing for ${renewal.runtimeId} after ${bindingReclaimAttempts} attempts: ${reclaimErr instanceof Error ? reclaimErr.message : String(reclaimErr)}`, ), ); } } else if (renewal && bindingReclaimAttempts > 0) { console.error( `[mx service] binding registry recovered without explicit reclaim for ${renewal.runtimeId} after ${bindingReclaimAttempts} attempt(s)`, ); bindingReclaimAttempts = 0; } } catch (err) { console.error( `[mx service] runtime presence tick failed: ${err instanceof Error ? err.message : String(err)}`, ); } })(); }, 10_000); const buildResult = (): IServiceCommandResult => ({ packageName: loaded.packageName, packageDir: loaded.packageDir, manifestPath: loaded.manifestPath, entryPath, exportName, checked: options.check === true, ...(selectedEnvironment ? { environmentName: selectedEnvironment.envName, environmentPath: selectedEnvironment.environmentPath, } : {}), ...(runnerTransport ? { runnerTransport } : {}), overrides: effectiveOverrides, ...(root ? { root } : {}), ...(mount ? { mount } : {}), ...(configRef ? { configRef } : {}), ...(secretsRef ? { secretsRef } : {}), instance: serializedInstance, serviceInstance: structuredClone(serializedInstance), deploymentInstance: structuredClone(serializedInstance), runtimeRegistration, bindingRegistration, ...(runtimeDeregistration ? { runtimeDeregistration } : {}), ...(bindingDeregistration ? { bindingDeregistration } : {}), result: sanitizeForJson(bootstrapResult), }); let shutdownPromise: Promise | null = null; return { result: buildResult(), ...(runnerRuntimeHandle.closed ? { transportClosed: runnerRuntimeHandle.closed } : {}), async shutdown(): Promise { if (!shutdownPromise) { shutdownPromise = (async () => { lifecycleState = 'stopping'; if (runtimePresenceTimer) { clearInterval(runtimePresenceTimer); } ({ runtimeDeregistration, bindingDeregistration } = await shutdownBootstrapRuntime( bootstrapResult, activeRuntime, runtimeRegistration, bindingRegistration, runnerRuntimeHandle, )); lifecycleState = 'stopped'; return buildResult(); })(); } return await shutdownPromise; }, shutdownRequested, }; } function registerHostHomeService( runtime: MatrixRuntime, matrixDir: string | undefined, ): void { const hostHome = matrixDir?.trim(); if (hostHome) { runtime.registerService('matrix-host-home', hostHome); runtime.registerService('credential-store', new HostCredentialStore(new HostAuthStateStore(hostHome))); } } async function mountMatrixHttpAssetEndpoint( runtime: MatrixRuntime, webapp: ILoadedMatrixWebappManifest, runtimeId?: string, ): Promise { const assetMount = buildMatrixHttpAssetMount(webapp.appName, runtimeId); if (!runtime.hasComponent(assetMount)) { await runtime.createSupervised(MatrixHttpAssetEndpointActor as unknown as new () => MatrixActor, assetMount, { assetEndpoint: { packageName: webapp.packageName, appName: webapp.appName, distDir: webapp.distDir, entryFile: webapp.entryFile, }, }); } return assetMount; } function buildRunnerWebappRouteMetadata( webapp: ILoadedMatrixWebappManifest, assetMount: string, ): IRunnerWebappRouteMetadata { return { appName: webapp.appName, routePrefix: `/apps/${webapp.appName}/`, ...(webapp.displayName ? { displayName: webapp.displayName } : {}), ...(webapp.icon ? { icon: webapp.icon } : {}), ...(typeof webapp.navOrder === 'number' ? { navOrder: webapp.navOrder } : {}), ...(webapp.description ? { description: webapp.description } : {}), ...(webapp.shells && webapp.shells.length > 0 ? { shells: webapp.shells } : {}), assetMount, }; } function parseOverrides(raw: string | undefined): Record { if (!raw) return {}; try { const parsed = JSON.parse(raw) as unknown; if (!isRecord(parsed)) { throw new Error('overrides must be a JSON object'); } return parsed; } catch (err) { throw new Error(`Failed to parse service overrides: ${err instanceof Error ? err.message : String(err)}`); } } function deepMerge( base: Record, patch: Record, ): Record { const output = structuredClone(base); for (const [key, value] of Object.entries(patch)) { const current = output[key]; if (isRecord(current) && isRecord(value)) { output[key] = deepMerge(current, value); continue; } output[key] = value; } return output; } function loadBootstrapEntryPath(packageDir: string, ref: string, label: string): string { const entryPath = ref.trim(); if (entryPath.length === 0) { throw new Error(`${label} must be a non-empty path`); } const filePath = resolveBootstrapRefPath(packageDir, entryPath); if (!fs.existsSync(filePath)) { throw new Error(`${label} not found at ${filePath}`); } return filePath; } async function shutdownBootstrapRuntime( result: unknown, activeRuntime?: MatrixRuntime, registration?: IRunnerRuntimeRegistration, bindingRegistration?: IRunnerBindingRegistration, runnerRuntimeHandle?: IRunnerRuntimeHandle, ): Promise<{ readonly runtimeDeregistration?: IRunnerRuntimeDeregistration; readonly bindingDeregistration?: IRunnerBindingDeregistration; }> { let shutdownHandled = false; let runtimeDeregistration: IRunnerRuntimeDeregistration | undefined; let bindingDeregistration: IRunnerBindingDeregistration | undefined; if (isRecord(result)) { const runtime: unknown = result.runtime; if (isRecord(runtime)) { const shutdown = runtime.shutdown; if (typeof shutdown === 'function') { if (activeRuntime) { bindingDeregistration = await deregisterRunnerBindings(activeRuntime, bindingRegistration); runtimeDeregistration = await deregisterRunnerRuntime(activeRuntime, registration); } await shutdown.call(runtime); shutdownHandled = Object.is(runtime, runnerRuntimeHandle?.runtime); } } } if (runnerRuntimeHandle && !shutdownHandled) { if (!runtimeDeregistration && activeRuntime) { bindingDeregistration = await deregisterRunnerBindings(activeRuntime, bindingRegistration); runtimeDeregistration = await deregisterRunnerRuntime(activeRuntime, registration); } await runnerRuntimeHandle.shutdown(); } return { ...(runtimeDeregistration ? { runtimeDeregistration } : {}), ...(bindingDeregistration ? { bindingDeregistration } : {}), }; } function resolveBootstrapRuntime(result: unknown): MatrixRuntime | undefined { if (!isRecord(result)) { return undefined; } const runtime = result.runtime; return runtime instanceof MatrixRuntime ? runtime : undefined; } function sanitizeForJson(value: unknown, depth = 8, seen = new WeakSet()): unknown { if (value == null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { return value; } if (typeof value === 'bigint') { return value.toString(); } if (typeof value === 'function' || typeof value === 'symbol') { return undefined; } if (Array.isArray(value)) { if (depth <= 0) return `[array:${value.length}]`; return value .map((entry) => sanitizeForJson(entry, depth - 1, seen)) .filter((entry) => entry !== undefined); } if (!isRecord(value)) { return String(value); } if (seen.has(value)) { return '[circular]'; } seen.add(value); if (depth <= 0) { return '[object]'; } const output: Record = {}; for (const [key, entry] of Object.entries(value)) { const sanitized = sanitizeForJson(entry, depth - 1, seen); if (sanitized !== undefined) { output[key] = sanitized; } } return output; } async function waitForShutdownSignal( onShutdown: () => Promise, shutdownRequested?: Promise, ): Promise { await new Promise((resolve, reject) => { let settled = false; const complete = (err?: unknown) => { if (settled) return; settled = true; process.off('SIGINT', handleSignal); process.off('SIGTERM', handleSignal); if (err) { reject(err); return; } resolve(); }; const handleSignal = () => { void onShutdown() .then(() => complete()) .catch((err) => complete(err)); }; process.on('SIGINT', handleSignal); process.on('SIGTERM', handleSignal); shutdownRequested ?.then(() => handleSignal()) .catch((err) => complete(err)); }); } function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); }