metro-file-map: Watch directories as the node crawler lists them

packages/metro-file-map/src/Watcher.js

@@ -8,6 +8,7 @@
  * @format
  */
 
+import type {CrawlWatcher} from './crawlers/node';
 import type {
   Console,
   Crawler,
@@ -48,6 +49,8 @@ const LIVE_CHECK_TIMEOUT = 200;
 const MAX_LIVE_WAIT_TIME = 10000;
 
 type InternalCrawlOptions = Readonly<{
+  // For a node crawl to report what it lists to, one for each root.
+  crawlWatchers?: ?ReadonlyArray<?CrawlWatcher>,
   previousState: CrawlerOptions['previousState'],
   roots: ReadonlyArray<string>,
   subpath?: string,
@@ -130,11 +133,27 @@ export class Watcher extends EventEmitter {
     // the same window. If the Watchman crawl falls back to the node crawler,
     // there is no clock, so #crawl() starts non-Watchman backends before the
     // node crawl instead.
+    //
+    // The node crawler lists every directory under the roots, so a
+    // FallbackWatcher started for it can watch each just before it is listed,
+    // instead of walking the roots again. It watches the paths the crawl
+    // lists, so only for roots given normalized.
+    let crawlWatchers: ?ReadonlyArray<?CrawlWatcher> = null;
     if (!options.useWatchman) {
-      await this.#startBackendsBeforeCrawl();
+      const crawlFed =
+        options.crawl == null &&
+        this.#getBackendClass() === FallbackWatcher &&
+        options.roots.every(root => path.resolve(root) === root);
+      await this.#startBackendsBeforeCrawl({crawlFed});
+      if (crawlFed) {
+        crawlWatchers = this.#backends.map(backend =>
+          backend instanceof FallbackWatcher ? backend.getCrawlWatcher() : null,
+        );
+      }
     }
 
     const result = await this.#crawl({
+      crawlWatchers,
       previousState: options.previousState,
       roots: options.roots,
       useWatchman: options.useWatchman,
@@ -144,11 +163,13 @@ export class Watcher extends EventEmitter {
     return result;
   }
 
-  async #startBackendsBeforeCrawl(): Promise<void> {
+  async #startBackendsBeforeCrawl(
+    opts?: Readonly<{crawlFed?: boolean}>,
+  ): Promise<void> {
     if (!this.#options.watch) {
       return;
     }
-    await this.#startBackends();
+    await this.#startBackends(undefined, opts?.crawlFed === true);
     // end() may have closed us before the backends were set - don't leak them.
     if (this.#options.abortSignal.aborted) {
       await this.close();
@@ -172,7 +193,7 @@ export class Watcher extends EventEmitter {
 
   async #crawl(crawlOptions: InternalCrawlOptions): Promise<CrawlResult> {
     const options = this.#options;
-    const {useWatchman, subpath} = crawlOptions;
+    const {crawlWatchers, useWatchman, subpath} = crawlOptions;
 
     const ignoreForCrawl = (filePath: string) =>
       options.ignoreForCrawl(filePath) ||
@@ -208,7 +229,11 @@ export class Watcher extends EventEmitter {
 
     let delta: CrawlResult;
     try {
-      delta = await crawl(crawlerOptions);
+      // Crawl watchers are only given for the node crawler.
+      delta =
+        crawlWatchers != null
+          ? await nodeCrawl(crawlerOptions, crawlWatchers)
+          : await crawl(crawlerOptions);
     } catch (firstError) {
       if (crawl !== watchmanCrawl) {
         throw firstError;
@@ -267,16 +292,23 @@ export class Watcher extends EventEmitter {
     }
   }
 
-  async #startBackends(clocks?: WatchmanClocks): Promise<void> {
-    const {extensions, ignorePatternForWatch} = this.#options;
-    const useWatchman = this.#useWatchmanWatcher;
-
+  #getBackendClass():
+    Class<WatchmanWatcher> | Class<NativeWatcher> | Class<FallbackWatcher> {
     // WatchmanWatcher > NativeWatcher > FallbackWatcher
-    const WatcherImpl = useWatchman
+    return this.#useWatchmanWatcher
       ? WatchmanWatcher
       : NativeWatcher.isSupported()
         ? NativeWatcher
         : FallbackWatcher;
+  }
+
+  async #startBackends(
+    clocks?: WatchmanClocks,
+    crawlFed?: boolean = false,
+  ): Promise<void> {
+    const {extensions, ignorePatternForWatch} = this.#options;
+    const useWatchman = this.#useWatchmanWatcher;
+    const WatcherImpl = this.#getBackendClass();
 
     let watcher = 'fallback';
     if (WatcherImpl === WatchmanWatcher) {
@@ -307,8 +339,10 @@ export class Watcher extends EventEmitter {
     const createWatcherBackend = (
       root: Path,
       rootClocks: ?Map<string, WatchmanClockSpec>,
+      crawlFed: boolean,
     ): Promise<WatcherBackend> => {
       const watcherOptions: WatcherBackendOptions = {
+        crawlFed,
         included: {
           extensions: new Set(extensions),
           // Ensure we always include package.json files, which are crucial for
@@ -366,10 +400,10 @@ export class Watcher extends EventEmitter {
 
     // A root added by watchRoot() is watched before it is crawled, so it
     // needs no crawl clock.
-    this.#createBackend = root => createWatcherBackend(root, null);
+    this.#createBackend = root => createWatcherBackend(root, null, false);
     this.#backends = await Promise.all(
       this.#options.roots.map(root =>
-        createWatcherBackend(root, watchmanClocks),
+        createWatcherBackend(root, watchmanClocks, crawlFed),
       ),
     );
     // Confirmed only once every backend has started, because starting one may

packages/metro-file-map/src/__tests__/Watcher-test.js

@@ -9,6 +9,7 @@
  * @oncall react_native
  */
 
+import type {CrawlWatcher} from '../crawlers/node';
 import type {
   Crawler,
   CrawlerOptions,
@@ -24,38 +25,65 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 
 const mockWatchmanCrawl = jest.fn<[CrawlerOptions], Promise<CrawlResult>>();
-const mockNodeCrawl = jest.fn<[CrawlerOptions], Promise<CrawlResult>>();
+const mockNodeCrawl = jest.fn<
+  [CrawlerOptions] | [CrawlerOptions, ReadonlyArray<?CrawlWatcher>],
+  Promise<CrawlResult>,
+>();
 jest.mock(
   '../crawlers/watchman',
   () => (options: CrawlerOptions) => mockWatchmanCrawl(options),
 );
 jest.mock(
   '../crawlers/node',
-  () => (options: CrawlerOptions) => mockNodeCrawl(options),
+  () =>
+    (options: CrawlerOptions, crawlWatchers?: ?ReadonlyArray<?CrawlWatcher>) =>
+      crawlWatchers != null
+        ? mockNodeCrawl(options, crawlWatchers)
+        : mockNodeCrawl(options),
 );
 
 type MockBackend = {
-  kind: 'native' | 'watchman',
+  kind: 'fallback' | 'native' | 'watchman',
   root: string,
   clocks: ?Map<string, string>,
+  crawlFed: boolean,
+  crawledDirectories: Array<string>,
   emit: (event: WatcherBackendChangeEvent) => void,
 };
 const mockStartedBackends: Array<MockBackend> = [];
 let mockNativeLiveOnStart = true;
+let mockNativeSupported = true;
 function mockBackendClass(kind: MockBackend['kind']) {
   return class {
+    readonly root: string;
     #root: string;
     #clocks: ?Map<string, string>;
+    #crawlFed: boolean;
+    #crawledDirectories: Array<string> = [];
     #listener: ?(event: WatcherBackendChangeEvent) => void;
     static isSupported(): boolean {
-      return true;
+      return kind !== 'native' || mockNativeSupported;
     }
     static isLiveOnStart(): boolean {
       return kind !== 'native' || mockNativeLiveOnStart;
     }
-    constructor(root: string, opts: {clocks?: ?Map<string, string>, ...}) {
+    constructor(
+      root: string,
+      opts: {clocks?: ?Map<string, string>, crawlFed?: boolean, ...},
+    ) {
+      this.root = root;
       this.#root = root;
       this.#clocks = opts.clocks;
+      this.#crawlFed = opts.crawlFed === true;
+    }
+    getCrawlWatcher(): CrawlWatcher {
+      return {
+        onDirectory: dir => {
+          this.#crawledDirectories.push(dir);
+        },
+        onSkippedDirectory: async () => {},
+        onSymlink: () => {},
+      };
     }
     onFileEvent(listener: (event: WatcherBackendChangeEvent) => void) {
       this.#listener = listener;
@@ -65,6 +93,8 @@ function mockBackendClass(kind: MockBackend['kind']) {
         kind,
         root: this.#root,
         clocks: this.#clocks,
+        crawlFed: this.#crawlFed,
+        crawledDirectories: this.#crawledDirectories,
         emit: event => this.#listener?.(event),
       });
     }
@@ -82,8 +112,13 @@ jest.mock('../watchers/WatchmanWatcher', () => ({
   __esModule: true,
   default: mockBackendClass('watchman'),
 }));
+jest.mock('../watchers/FallbackWatcher', () => ({
+  __esModule: true,
+  default: mockBackendClass('fallback'),
+}));
 
-const rootDir = path.join(path.sep, 'project');
+// Resolved, as roots are in practice (on Windows, with a drive letter)
+const rootDir = path.resolve(path.sep, 'project');
 
 function crawlResult(canonicalPaths: ReadonlyArray<string>): CrawlResult {
   const changedFiles: FileData = new Map();
@@ -142,6 +177,7 @@ function createWatcher(
 beforeEach(() => {
   mockStartedBackends.length = 0;
   mockNativeLiveOnStart = true;
+  mockNativeSupported = true;
   mockWatchmanCrawl.mockReset();
   mockNodeCrawl.mockReset();
   jest.spyOn(console, 'warn').mockImplementation(() => {});
@@ -388,6 +424,59 @@ describe('Watcher with a backend that is not live on start', () => {
   });
 });
 
+describe('Watcher with FallbackWatcher and the node crawler', () => {
+  beforeEach(() => {
+    mockNativeSupported = false;
+  });
+
+  test('watches what the crawl lists, instead of walking the roots', async () => {
+    const src = path.join(rootDir, 'src');
+    mockNodeCrawl.mockImplementation(async (options, crawlWatchers) => {
+      // Started, and watching the root, before the crawl
+      expect(mockStartedBackends.map(b => [b.kind, b.crawlFed])).toEqual([
+        ['fallback', true],
+      ]);
+      expect(crawlWatchers).toHaveLength(1);
+      crawlWatchers?.[0]?.onDirectory(src);
+      return crawlResult([]);
+    });
+    const watcher = createWatcher(false, null, true);
+
+    await watcher.crawl();
+
+    expect(mockStartedBackends[0].crawledDirectories).toEqual([src]);
+  });
+
+  test('a scoped recrawl is not reported to the watcher', async () => {
+    mockNodeCrawl.mockImplementation(async () => crawlResult([]));
+    const watcher = createWatcher(false, null, true);
+    await watcher.crawl();
+    mockNodeCrawl.mockClear();
+
+    await watcher.recrawl(
+      'src',
+      new TreeFS({
+        processFile: () => {
+          throw new Error('not implemented');
+        },
+        rootDir,
+      }),
+    );
+
+    expect(mockNodeCrawl.mock.calls).toEqual([[expect.any(Object)]]);
+  });
+
+  test('with a custom crawler, the watcher walks the roots itself', async () => {
+    const watcher = createWatcher(false, async () => crawlResult([]), true);
+
+    await watcher.crawl();
+
+    expect(mockStartedBackends.map(b => [b.kind, b.crawlFed])).toEqual([
+      ['fallback', false],
+    ]);
+  });
+});
+
 describe('Watcher crawl option', () => {
   const stubCrawler =
     (canonicalPaths: ReadonlyArray<string>): Crawler =>

packages/metro-file-map/src/__tests__/addRoot-test.js

@@ -9,6 +9,7 @@
  * @oncall react_native
  */
 
+import type {CrawlWatcher} from '../crawlers/node';
 import type {
   CacheData,
   ChangeEvent,
@@ -90,6 +91,14 @@ class MockWatcher extends AbstractWatcher {
   static isSupported(): boolean {
     return true;
   }
+  // Stands in for FallbackWatcher too, which a node crawl may report to
+  getCrawlWatcher(): CrawlWatcher {
+    return {
+      onDirectory: () => {},
+      onSkippedDirectory: async () => {},
+      onSymlink: () => {},
+    };
+  }
 }
 
 jest.mock('../watchers/FallbackWatcher', () => MockWatcher);

packages/metro-file-map/src/__tests__/index-test.js

@@ -10,6 +10,7 @@
  */
 
 import type {InputOptions} from '..';
+import type {CrawlWatcher} from '../crawlers/node';
 import type {
   BuildResult,
   CanonicalPath,
@@ -156,6 +157,14 @@ class MockWatcher extends AbstractWatcher {
   static isSupported(): boolean {
     return true;
   }
+  // Stands in for FallbackWatcher too, which a node crawl may report to
+  getCrawlWatcher(): CrawlWatcher {
+    return {
+      onDirectory: () => {},
+      onSkippedDirectory: async () => {},
+      onSymlink: () => {},
+    };
+  }
 }
 
 jest.mock('../watchers/FallbackWatcher', () => MockWatcher);

packages/metro-file-map/src/__tests__/watch-integration-test.js

@@ -172,4 +172,17 @@ describe('FileMap in watch mode', () => {
     await until('marker.js added', () => exists('src/marker.js'));
     expect(exists('ignored-dir/y.js')).toBe(false);
   });
+
+  // The crawl skips node_modules without `retainAllFiles`, but changes in it
+  // are still reported in watch mode.
+  test('watches a directory the crawl skips', async () => {
+    write('node_modules/pkg/index.js');
+    await fileMap.end();
+    fileMap = createFileMap({retainAllFiles: false});
+    ({fileSystem} = await fileMap.build());
+    expect(exists('node_modules/pkg/index.js')).toBe(false);
+
+    write('node_modules/pkg/new.js');
+    await until('new.js added', () => exists('node_modules/pkg/new.js'));
+  });
 });