Skip to content

Commit 69ecb99

Browse files
arcanisarcanis
authored
Implements the readdir recursive flag (#5514)
**What's the problem this PR addresses?** The `ZipFS` implementation doesn't support the `recursive` flag. Fixes part of #5423 **How did you fix it?** I implemented the `recursive` flag and updated the types to account for it. However I didn't make `ZipOpenFS` automatically traverse through zip archives ... My line of thinking was: > - on one hand, tools implementing their own readdir-based recursive traversal don't currently dig into zip archives > - on the other hand, it can be argued that the other recursive Node.js native operations also apply to the zip archives' content (`cp` / `rm`) > > But I tend to think the first point is more important ... I feel like the recursive flag should just be an optimization, so it should have the same behaviour as a manual traversal ... **Checklist** <!--- Don't worry if you miss something, chores are automatically tested. --> <!--- This checklist exists to help you remember doing the chores when you submit a PR. --> <!--- Put an `x` in all the boxes that apply. --> - [x] I have read the [Contributing Guide](https://yarnpkg.com/advanced/contributing). <!-- See https://yarnpkg.com/advanced/contributing#preparing-your-pr-to-be-released for more details. --> <!-- Check with `yarn version check` and fix with `yarn version check -i` --> - [x] I have set the packages that need to be released for my changes to be effective. <!-- The "Testing chores" workflow validates that your PR follows our guidelines. --> <!-- If it doesn't pass, click on it to see details as to what your PR might be missing. --> - [x] I will check that all automated PR checks pass before the PR gets reviewed.
1 parent 2259237 commit 69ecb99

File tree

13 files changed

+394
-119
lines changed

13 files changed

+394
-119
lines changed

‎.yarn/versions/cc34eb37.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
releases:
2+
"@yarnpkg/cli": minor
3+
"@yarnpkg/core": minor
4+
"@yarnpkg/fslib": minor
5+
"@yarnpkg/libzip": minor
6+
"@yarnpkg/plugin-pnpm": minor
7+
"@yarnpkg/pnpify": minor
8+
9+
declined:
10+
- "@yarnpkg/plugin-compat"
11+
- "@yarnpkg/plugin-constraints"
12+
- "@yarnpkg/plugin-dlx"
13+
- "@yarnpkg/plugin-essentials"
14+
- "@yarnpkg/plugin-exec"
15+
- "@yarnpkg/plugin-file"
16+
- "@yarnpkg/plugin-git"
17+
- "@yarnpkg/plugin-github"
18+
- "@yarnpkg/plugin-http"
19+
- "@yarnpkg/plugin-init"
20+
- "@yarnpkg/plugin-interactive-tools"
21+
- "@yarnpkg/plugin-link"
22+
- "@yarnpkg/plugin-nm"
23+
- "@yarnpkg/plugin-npm"
24+
- "@yarnpkg/plugin-npm-cli"
25+
- "@yarnpkg/plugin-pack"
26+
- "@yarnpkg/plugin-patch"
27+
- "@yarnpkg/plugin-pnp"
28+
- "@yarnpkg/plugin-stage"
29+
- "@yarnpkg/plugin-typescript"
30+
- "@yarnpkg/plugin-version"
31+
- "@yarnpkg/plugin-workspace-tools"
32+
- vscode-zipfs
33+
- "@yarnpkg/builder"
34+
- "@yarnpkg/doctor"
35+
- "@yarnpkg/extensions"
36+
- "@yarnpkg/nm"
37+
- "@yarnpkg/pnp"
38+
- "@yarnpkg/sdks"
39+
- "@yarnpkg/shell"

‎packages/plugin-pnpm/sources/PnpmLinker.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import {Descriptor, FetchResult, formatUtils, Installer, InstallPackageExtraApi, Linker, LinkOptions, LinkType, Locator, LocatorHash, Manifest, MessageName, MinimalLinkOptions, Package, Project, miscUtils, structUtils, WindowsLinkType} from '@yarnpkg/core';
2-
import {Dirent, Filename, PortablePath, setupCopyIndex, ppath, xfs} from '@yarnpkg/fslib';
2+
import {Filename, PortablePath, setupCopyIndex, ppath, xfs, DirentNoPath} from '@yarnpkg/fslib';
33
import {jsInstallUtils} from '@yarnpkg/plugin-pnp';
44
import {UsageError} from 'clipanion';
55

@@ -339,9 +339,9 @@ function isPnpmVirtualCompatible(locator: Locator, {project}: {project: Project}
339339
}
340340

341341
async function getNodeModulesListing(nmPath: PortablePath) {
342-
const listing = new Map<PortablePath, Dirent>();
342+
const listing = new Map<PortablePath, DirentNoPath>();
343343

344-
let fsListing: Array<Dirent> = [];
344+
let fsListing: Array<DirentNoPath> = [];
345345
try {
346346
fsListing = await xfs.readdirPromise(nmPath, {withFileTypes: true});
347347
} catch (err) {
@@ -377,7 +377,7 @@ async function getNodeModulesListing(nmPath: PortablePath) {
377377
return listing;
378378
}
379379

380-
async function cleanNodeModules(nmPath: PortablePath, extraneous: Map<PortablePath, Dirent>) {
380+
async function cleanNodeModules(nmPath: PortablePath, extraneous: Map<PortablePath, DirentNoPath>) {
381381
const removeNamePromises = [];
382382
const scopesToRemove = new Set<Filename>();
383383

‎packages/yarnpkg-fslib/sources/FakeFS.ts

+39-14
Original file line numberDiff line numberDiff line change
@@ -18,28 +18,39 @@ export type BigIntStats = NodeBigIntStats & {
1818
crc?: number;
1919
};
2020

21-
export type Dirent = Exclude<NodeDirent, 'name'> & {
21+
export type Dirent<T extends Path> = Exclude<NodeDirent, 'name'> & {
22+
name: Filename;
23+
path: T;
24+
};
25+
26+
export type DirentNoPath = Exclude<NodeDirent, 'name'> & {
2227
name: Filename;
2328
};
2429

2530
export type Dir<P extends Path> = {
2631
readonly path: P;
2732

28-
[Symbol.asyncIterator](): AsyncIterableIterator<Dirent>;
33+
[Symbol.asyncIterator](): AsyncIterableIterator<DirentNoPath>;
2934

3035
close(): Promise<void>;
3136
close(cb: NoParamCallback): void;
3237

3338
closeSync(): void;
3439

35-
read(): Promise<Dirent | null>;
36-
read(cb: (err: NodeJS.ErrnoException | null, dirent: Dirent | null) => void): void;
40+
read(): Promise<DirentNoPath | null>;
41+
read(cb: (err: NodeJS.ErrnoException | null, dirent: DirentNoPath | null) => void): void;
3742

38-
readSync(): Dirent | null;
43+
readSync(): DirentNoPath | null;
3944
};
4045

4146
export type OpendirOptions = Partial<{
4247
bufferSize: number;
48+
recursive: boolean;
49+
}>;
50+
51+
export type ReaddirOptions = Partial<{
52+
recursive: boolean;
53+
withFileTypes: boolean;
4354
}>;
4455

4556
export type CreateReadStreamOptions = Partial<{
@@ -161,15 +172,29 @@ export abstract class FakeFS<P extends Path> {
161172
abstract realpathPromise(p: P): Promise<P>;
162173
abstract realpathSync(p: P): P;
163174

164-
abstract readdirPromise(p: P): Promise<Array<Filename>>;
165-
abstract readdirPromise(p: P, opts: {withFileTypes: false} | null): Promise<Array<Filename>>;
166-
abstract readdirPromise(p: P, opts: {withFileTypes: true}): Promise<Array<Dirent>>;
167-
abstract readdirPromise(p: P, opts: {withFileTypes: boolean}): Promise<Array<Filename> | Array<Dirent>>;
168-
169-
abstract readdirSync(p: P): Array<Filename>;
170-
abstract readdirSync(p: P, opts: {withFileTypes: false} | null): Array<Filename>;
171-
abstract readdirSync(p: P, opts: {withFileTypes: true}): Array<Dirent>;
172-
abstract readdirSync(p: P, opts: {withFileTypes: boolean}): Array<Filename> | Array<Dirent>;
175+
abstract readdirPromise(p: P, opts?: null): Promise<Array<Filename>>;
176+
abstract readdirPromise(p: P, opts: {recursive?: false, withFileTypes: true}): Promise<Array<DirentNoPath>>;
177+
abstract readdirPromise(p: P, opts: {recursive?: false, withFileTypes?: false}): Promise<Array<Filename>>;
178+
abstract readdirPromise(p: P, opts: {recursive?: false, withFileTypes: boolean}): Promise<Array<DirentNoPath | Filename>>;
179+
abstract readdirPromise(p: P, opts: {recursive: true, withFileTypes: true}): Promise<Array<Dirent<P>>>;
180+
abstract readdirPromise(p: P, opts: {recursive: true, withFileTypes?: false}): Promise<Array<P>>;
181+
abstract readdirPromise(p: P, opts: {recursive: true, withFileTypes: boolean}): Promise<Array<Dirent<P> | P>>;
182+
abstract readdirPromise(p: P, opts: {recursive: boolean, withFileTypes: true}): Promise<Array<Dirent<P> | DirentNoPath>>;
183+
abstract readdirPromise(p: P, opts: {recursive: boolean, withFileTypes?: false}): Promise<Array<P>>;
184+
abstract readdirPromise(p: P, opts: {recursive: boolean, withFileTypes: boolean}): Promise<Array<Dirent<P> | DirentNoPath | P>>;
185+
abstract readdirPromise(p: P, opts?: ReaddirOptions | null): Promise<Array<Dirent<P> | DirentNoPath | P>>;
186+
187+
abstract readdirSync(p: P, opts?: null): Array<Filename>;
188+
abstract readdirSync(p: P, opts: {recursive?: false, withFileTypes: true}): Array<DirentNoPath>;
189+
abstract readdirSync(p: P, opts: {recursive?: false, withFileTypes?: false}): Array<Filename>;
190+
abstract readdirSync(p: P, opts: {recursive?: false, withFileTypes: boolean}): Array<DirentNoPath | Filename>;
191+
abstract readdirSync(p: P, opts: {recursive: true, withFileTypes: true}): Array<Dirent<P>>;
192+
abstract readdirSync(p: P, opts: {recursive: true, withFileTypes?: false}): Array<P>;
193+
abstract readdirSync(p: P, opts: {recursive: true, withFileTypes: boolean}): Array<Dirent<P> | P>;
194+
abstract readdirSync(p: P, opts: {recursive: boolean, withFileTypes: true}): Array<Dirent<P> | DirentNoPath>;
195+
abstract readdirSync(p: P, opts: {recursive: boolean, withFileTypes?: false}): Array<P>;
196+
abstract readdirSync(p: P, opts: {recursive: boolean, withFileTypes: boolean}): Array<Dirent<P> | DirentNoPath | P>;
197+
abstract readdirSync(p: P, opts?: ReaddirOptions | null): Array<Dirent<P> | DirentNoPath | P>;
173198

174199
abstract existsPromise(p: P): Promise<boolean>;
175200
abstract existsSync(p: P): boolean;

‎packages/yarnpkg-fslib/sources/MountFS.ts

+23-11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import {BigIntStats, constants, Stats} from 'fs';
22

3-
import {WatchOptions, WatchCallback, Watcher, StatOptions, StatSyncOptions} from './FakeFS';
3+
import {WatchOptions, WatchCallback, Watcher, StatOptions, StatSyncOptions, ReaddirOptions, DirentNoPath} from './FakeFS';
44
import {FakeFS, MkdirOptions, RmdirOptions, WriteFileOptions, OpendirOptions} from './FakeFS';
55
import {Dirent, SymlinkType} from './FakeFS';
66
import {CreateReadStreamOptions, CreateWriteStreamOptions, BasePortableFakeFS, ExtractHintOptions, WatchFileOptions, WatchFileCallback, StatWatcher} from './FakeFS';
@@ -806,11 +806,17 @@ export class MountFS<MountedFS extends MountableFS> extends BasePortableFakeFS {
806806
});
807807
}
808808

809-
async readdirPromise(p: PortablePath): Promise<Array<Filename>>;
810-
async readdirPromise(p: PortablePath, opts: {withFileTypes: false} | null): Promise<Array<Filename>>;
811-
async readdirPromise(p: PortablePath, opts: {withFileTypes: true}): Promise<Array<Dirent>>;
812-
async readdirPromise(p: PortablePath, opts: {withFileTypes: boolean}): Promise<Array<Filename> | Array<Dirent>>;
813-
async readdirPromise(p: PortablePath, opts?: {withFileTypes?: boolean} | null): Promise<Array<string> | Array<Dirent>> {
809+
async readdirPromise(p: PortablePath, opts?: null): Promise<Array<Filename>>;
810+
async readdirPromise(p: PortablePath, opts: {recursive?: false, withFileTypes: true}): Promise<Array<DirentNoPath>>;
811+
async readdirPromise(p: PortablePath, opts: {recursive?: false, withFileTypes?: false}): Promise<Array<Filename>>;
812+
async readdirPromise(p: PortablePath, opts: {recursive?: false, withFileTypes: boolean}): Promise<Array<DirentNoPath | Filename>>;
813+
async readdirPromise(p: PortablePath, opts: {recursive: true, withFileTypes: true}): Promise<Array<Dirent<PortablePath>>>;
814+
async readdirPromise(p: PortablePath, opts: {recursive: true, withFileTypes?: false}): Promise<Array<PortablePath>>;
815+
async readdirPromise(p: PortablePath, opts: {recursive: true, withFileTypes: boolean}): Promise<Array<Dirent<PortablePath> | PortablePath>>;
816+
async readdirPromise(p: PortablePath, opts: {recursive: boolean, withFileTypes: true}): Promise<Array<Dirent<PortablePath> | DirentNoPath>>;
817+
async readdirPromise(p: PortablePath, opts: {recursive: boolean, withFileTypes?: false}): Promise<Array<PortablePath>>;
818+
async readdirPromise(p: PortablePath, opts: {recursive: boolean, withFileTypes: boolean}): Promise<Array<Dirent<PortablePath> | DirentNoPath | PortablePath>>;
819+
async readdirPromise(p: PortablePath, opts?: ReaddirOptions | null): Promise<Array<Dirent<PortablePath> | DirentNoPath | PortablePath>> {
814820
return await this.makeCallPromise(p, async () => {
815821
return await this.baseFs.readdirPromise(p, opts as any);
816822
}, async (mountFs, {subPath}) => {
@@ -820,11 +826,17 @@ export class MountFS<MountedFS extends MountableFS> extends BasePortableFakeFS {
820826
});
821827
}
822828

823-
readdirSync(p: PortablePath): Array<Filename>;
824-
readdirSync(p: PortablePath, opts: {withFileTypes: false} | null): Array<Filename>;
825-
readdirSync(p: PortablePath, opts: {withFileTypes: true}): Array<Dirent>;
826-
readdirSync(p: PortablePath, opts: {withFileTypes: boolean}): Array<Filename> | Array<Dirent>;
827-
readdirSync(p: PortablePath, opts?: {withFileTypes?: boolean} | null): Array<string> | Array<Dirent> {
829+
readdirSync(p: PortablePath, opts?: null): Array<Filename>;
830+
readdirSync(p: PortablePath, opts: {recursive?: false, withFileTypes: true}): Array<DirentNoPath>;
831+
readdirSync(p: PortablePath, opts: {recursive?: false, withFileTypes?: false}): Array<Filename>;
832+
readdirSync(p: PortablePath, opts: {recursive?: false, withFileTypes: boolean}): Array<DirentNoPath | Filename>;
833+
readdirSync(p: PortablePath, opts: {recursive: true, withFileTypes: true}): Array<Dirent<PortablePath>>;
834+
readdirSync(p: PortablePath, opts: {recursive: true, withFileTypes?: false}): Array<PortablePath>;
835+
readdirSync(p: PortablePath, opts: {recursive: true, withFileTypes: boolean}): Array<Dirent<PortablePath> | PortablePath>;
836+
readdirSync(p: PortablePath, opts: {recursive: boolean, withFileTypes: true}): Array<Dirent<PortablePath> | DirentNoPath>;
837+
readdirSync(p: PortablePath, opts: {recursive: boolean, withFileTypes?: false}): Array<PortablePath>;
838+
readdirSync(p: PortablePath, opts: {recursive: boolean, withFileTypes: boolean}): Array<Dirent<PortablePath> | DirentNoPath | PortablePath>;
839+
readdirSync(p: PortablePath, opts?: ReaddirOptions | null): Array<Dirent<PortablePath> | DirentNoPath | PortablePath> {
828840
return this.makeCallSync(p, () => {
829841
return this.baseFs.readdirSync(p, opts as any);
830842
}, (mountFs, {subPath}) => {

‎packages/yarnpkg-fslib/sources/NodeFS.ts

+31-19
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
import fs, {BigIntStats, Stats} from 'fs';
1+
import fs, {BigIntStats, Stats} from 'fs';
22

3-
import {CreateReadStreamOptions, CreateWriteStreamOptions, Dir, StatWatcher, WatchFileCallback, WatchFileOptions, OpendirOptions} from './FakeFS';
4-
import {Dirent, SymlinkType, StatSyncOptions, StatOptions} from './FakeFS';
5-
import {BasePortableFakeFS, WriteFileOptions} from './FakeFS';
6-
import {MkdirOptions, RmdirOptions, WatchOptions, WatchCallback, Watcher} from './FakeFS';
7-
import {FSPath, PortablePath, Filename, ppath, npath} from './path';
3+
import {CreateReadStreamOptions, CreateWriteStreamOptions, Dir, StatWatcher, WatchFileCallback, WatchFileOptions, OpendirOptions, ReaddirOptions, DirentNoPath} from './FakeFS';
4+
import {Dirent, SymlinkType, StatSyncOptions, StatOptions} from './FakeFS';
5+
import {BasePortableFakeFS, WriteFileOptions} from './FakeFS';
6+
import {MkdirOptions, RmdirOptions, WatchOptions, WatchCallback, Watcher} from './FakeFS';
7+
import {FSPath, PortablePath, Filename, ppath, npath} from './path';
88

99
export class NodeFS extends BasePortableFakeFS {
1010
private readonly realFs: typeof fs;
@@ -422,12 +422,18 @@ export class NodeFS extends BasePortableFakeFS {
422422
return this.realFs.readFileSync(fsNativePath, encoding);
423423
}
424424

425-
async readdirPromise(p: PortablePath): Promise<Array<Filename>>;
426-
async readdirPromise(p: PortablePath, opts: {withFileTypes: false} | null): Promise<Array<Filename>>;
427-
async readdirPromise(p: PortablePath, opts: {withFileTypes: true}): Promise<Array<Dirent>>;
428-
async readdirPromise(p: PortablePath, opts: {withFileTypes: boolean}): Promise<Array<Filename> | Array<Dirent>>;
429-
async readdirPromise(p: PortablePath, opts?: {withFileTypes?: boolean} | null): Promise<Array<string> | Array<Dirent>> {
430-
return await new Promise<Array<Filename> | Array<Dirent>>((resolve, reject) => {
425+
async readdirPromise(p: PortablePath, opts?: null): Promise<Array<Filename>>;
426+
async readdirPromise(p: PortablePath, opts: {recursive?: false, withFileTypes: true}): Promise<Array<DirentNoPath>>;
427+
async readdirPromise(p: PortablePath, opts: {recursive?: false, withFileTypes?: false}): Promise<Array<Filename>>;
428+
async readdirPromise(p: PortablePath, opts: {recursive?: false, withFileTypes: boolean}): Promise<Array<DirentNoPath | Filename>>;
429+
async readdirPromise(p: PortablePath, opts: {recursive: true, withFileTypes: true}): Promise<Array<Dirent<PortablePath>>>;
430+
async readdirPromise(p: PortablePath, opts: {recursive: true, withFileTypes?: false}): Promise<Array<PortablePath>>;
431+
async readdirPromise(p: PortablePath, opts: {recursive: true, withFileTypes: boolean}): Promise<Array<Dirent<PortablePath> | PortablePath>>;
432+
async readdirPromise(p: PortablePath, opts: {recursive: boolean, withFileTypes: true}): Promise<Array<Dirent<PortablePath> | DirentNoPath>>;
433+
async readdirPromise(p: PortablePath, opts: {recursive: boolean, withFileTypes?: false}): Promise<Array<PortablePath>>;
434+
async readdirPromise(p: PortablePath, opts: {recursive: boolean, withFileTypes: boolean}): Promise<Array<Dirent<PortablePath> | DirentNoPath | PortablePath>>;
435+
async readdirPromise(p: PortablePath, opts?: ReaddirOptions | null): Promise<Array<Dirent<PortablePath> | DirentNoPath | PortablePath>> {
436+
return await new Promise<any>((resolve, reject) => {
431437
if (opts?.withFileTypes) {
432438
this.realFs.readdir(npath.fromPortablePath(p), {withFileTypes: true}, this.makeCallback(resolve, reject) as any);
433439
} else {
@@ -436,15 +442,21 @@ export class NodeFS extends BasePortableFakeFS {
436442
});
437443
}
438444

439-
readdirSync(p: PortablePath): Array<Filename>;
440-
readdirSync(p: PortablePath, opts: {withFileTypes: false} | null): Array<Filename>;
441-
readdirSync(p: PortablePath, opts: {withFileTypes: true}): Array<Dirent>;
442-
readdirSync(p: PortablePath, opts: {withFileTypes: boolean}): Array<Filename> | Array<Dirent>;
443-
readdirSync(p: PortablePath, opts?: {withFileTypes?: boolean} | null): Array<string> | Array<Dirent> {
445+
readdirSync(p: PortablePath, opts?: null): Array<Filename>;
446+
readdirSync(p: PortablePath, opts: {recursive?: false, withFileTypes: true}): Array<DirentNoPath>;
447+
readdirSync(p: PortablePath, opts: {recursive?: false, withFileTypes?: false}): Array<Filename>;
448+
readdirSync(p: PortablePath, opts: {recursive?: false, withFileTypes: boolean}): Array<DirentNoPath | Filename>;
449+
readdirSync(p: PortablePath, opts: {recursive: true, withFileTypes: true}): Array<Dirent<PortablePath>>;
450+
readdirSync(p: PortablePath, opts: {recursive: true, withFileTypes?: false}): Array<PortablePath>;
451+
readdirSync(p: PortablePath, opts: {recursive: true, withFileTypes: boolean}): Array<Dirent<PortablePath> | PortablePath>;
452+
readdirSync(p: PortablePath, opts: {recursive: boolean, withFileTypes: true}): Array<Dirent<PortablePath> | DirentNoPath>;
453+
readdirSync(p: PortablePath, opts: {recursive: boolean, withFileTypes?: false}): Array<PortablePath>;
454+
readdirSync(p: PortablePath, opts: {recursive: boolean, withFileTypes: boolean}): Array<Dirent<PortablePath> | DirentNoPath | PortablePath>;
455+
readdirSync(p: PortablePath, opts?: ReaddirOptions | null): Array<Dirent<PortablePath> | DirentNoPath | PortablePath> {
444456
if (opts?.withFileTypes) {
445-
return this.realFs.readdirSync(npath.fromPortablePath(p), {withFileTypes: true} as any);
457+
return this.realFs.readdirSync(npath.fromPortablePath(p), {withFileTypes: true} as any) as Array<any>;
446458
} else {
447-
return this.realFs.readdirSync(npath.fromPortablePath(p)) as Array<Filename>;
459+
return this.realFs.readdirSync(npath.fromPortablePath(p)) as Array<any>;
448460
}
449461
}
450462

0 commit comments

Comments
 (0)