343def batch_index_dataset(
344    roots: Sequence[str | PathT],
345    max_workers: int | None = 0,
346    executor_cls: type[ProcessPoolExecutor | ThreadPoolExecutor] = ProcessPoolExecutor,
347    *,
348    filters: dict[str, str | list[str]] | None = None,
349    show_progress: bool = False,
350    schema: SchemaSpec = None,
351) -> Generator[pa.Table, None, None]:
352    """Index a batch of BIDS datasets.
353
354    Args:
355        roots: List of BIDS dataset root directories.
356        max_workers: Number of indexing processes to run in parallel. Setting
357            `max_workers=0` (the default) uses the main process only. Setting
358            `max_workers=None` starts as many workers as there are available CPUs. See
359            `concurrent.futures.ProcessPoolExecutor` for details.
360        executor_cls: Executor class to use for parallel indexing.
361        filters: Dict mapping entity keys to glob patterns or lists of patterns.
362        show_progress: Show progress bar.
363        schema: Optional `SchemaSpec`. `None` uses the default BIDS schema.
364
365    Yields:
366        An Arrow table index for each BIDS dataset.
367    """
368    func = partial(_batch_index_func, filters=filters, schema=schema)
369    file_count = 0
370    for dataset, table in (
371        pbar := tqdm(
372            _pmap(func, roots, max_workers, executor_cls=executor_cls),
373            total=len(roots) if isinstance(roots, Sequence) else None,
374            disable=show_progress not in {True, "dataset"},
375        )
376    ):
377        file_count += len(table)
378        pbar.set_postfix({"ds": dataset, "N": _hfmt(file_count)}, refresh=False)
379        yield table

136def clear_schema_caches() -> None:
137    """Clear the LRU caches of the schema-dependent lookup functions.
138
139    Call to force recomputation when cached inputs change, or to release
140    memory after a large indexing run.
141
142    This also clears ``bidsschematools.schema.load_schema``'s own cache,
143    since a path-based schema is ultimately resolved through it; without that,
144    a changed schema file would be served from that cache even after the
145    bids2table caches above are cleared.
146    """
147    bidsschematools.schema.load_schema.cache_clear()
148    _load_from_path.cache_clear()
149    entity_arrow_schema.cache_clear()
150    _lookups_from_arrow.cache_clear()
151    _build_datatype_pattern.cache_clear()
152    _cache_parse_bids_entities.cache_clear()
153    _is_bids_dataset.cache_clear()
154    _get_bids_dataset.cache_clear()
155    _load_bidsignore_patterns.cache_clear()
156    _read_dataset_description.cache_clear()
157    get_entity_directory_order.cache_clear()
158    get_json_data_suffixes.cache_clear()
159    get_file_entity_prefixes.cache_clear()

38def cloudpathlib_is_available() -> bool:
39    """Check if cloudpathlib is available.
40
41    Returns:
42        ``True`` if the ``cloud`` extra (cloudpathlib) is installed, else ``False``.
43    """
44    return _CLOUDPATHLIB_AVAILABLE

218def find_bids_datasets(
219    root: str | PathT,
220    exclude: str | list[str] | None = None,
221    maxdepth: int | None = None,
222    *,
223    schema: SchemaSpec = None,
224) -> Generator[PathT, None, None]:
225    """Find all BIDS datasets under a root directory.
226
227    Args:
228        root: Root path to begin search.
229        exclude: Glob pattern or list of patterns matching sub-directory names to
230            exclude from the search.
231        maxdepth: Maximum depth to search.
232        schema: BIDS schema specification to use. If ``None``, uses the bundled
233            default schema.
234
235    Yields:
236        Root paths of all BIDS datasets under `root`.
237    """
238    root = as_path(root)
239    adapter = load_bids_schema(schema)
240
241    if isinstance(exclude, str):
242        exclude = [exclude]
243    elif exclude is None:
244        exclude = []
245    exclude_patterns = [re.compile(fnmatch.translate(pat)) for pat in exclude]
246
247    entry_count = 1
248    ds_count = 0
249
250    if _is_bids_dataset(root, adapter):
251        ds_count += 1
252        yield root
253
254    stack = [(root, 0)]
255
256    while stack:
257        top, depth = stack.pop()
258
259        inside_bids = _is_bids_dataset(top, adapter)
260        depth += 1
261
262        for entry in top.iterdir():
263            entry_count += 1
264
265            if any(re.fullmatch(pat, entry.name) for pat in exclude_patterns):
266                continue
267
268            if _is_bids_dataset(entry, adapter):
269                ds_count += 1
270                yield entry
271
272            # Checks if we should descend into this directory (not reach final depth).
273            descend = maxdepth is None or depth < maxdepth
274            # Heuristic checks whether the filename looks like a (visible) directory.
275            descend = descend and not (entry.suffix or entry.name.startswith("."))
276            # Only descend into specific subdirectories of BIDS directories.
277            descend = descend and (
278                not inside_bids or entry.name in _BIDS_NESTED_PARENT_DIRNAMES
279            )
280            # Check if actually a directory (slow, so short-circuit if possible).
281            if descend and entry.is_dir():
282                stack.append((entry, depth))

215def format_bids_path(
216    entities: dict[str, Any], int_format: str = "%d", schema: SchemaSpec = None
217) -> Path:
218    """Construct a formatted BIDS path from an entities dict.
219
220    Directory entities (e.g. ``sub``, ``ses``, ``tpl``) become path
221    directories and are also repeated in the file name, per the BIDS
222    convention. The special entities are handled by their role: ``datatype``
223    is the innermost directory, while ``suffix`` and ``ext`` are appended to
224    the file name. All other entities are formatted into the file name.
225
226    Args:
227        entities: dict mapping BIDS entity names to values.
228        int_format: format string for integer (index) BIDS values.
229        schema: optional BIDS schema. If ``None``, uses the default schema.
230
231    Returns:
232        A formatted `Path` instance.
233    """
234    adapter = load_bids_schema(schema)
235    dir_order = get_entity_directory_order(adapter)
236    special = {
237        cfg.get("name", entity)
238        for entity, cfg in adapter.entity_schema.items()
239        if cfg.get("format") == "special"
240    }
241
242    name_parts = []
243    for name, value in entities.items():
244        if name not in special:
245            if isinstance(value, int):
246                value = int_format % value
247            name_parts.append(f"{name}-{value}")
248    name = "_".join(name_parts)
249
250    if suffix := entities.get("suffix"):
251        name += f"_{suffix}"
252    if ext := entities.get("ext"):
253        name += ext
254
255    # Prepend parent directories, innermost to outermost.
256    path = Path(name)
257    if datatype := entities.get("datatype"):
258        path = Path(datatype) / path
259    for dir_entity in reversed(dir_order):
260        if dir_entity in entities:
261            path = Path(f"{dir_entity}-{entities[dir_entity]}") / path
262    return path

162def get_arrow_schema(*, schema: SchemaSpec | BIDSSchemaAdapter = None) -> pa.Schema:
163    """Get Arrow schema of the BIDS dataset index.
164
165    Args:
166        schema: BIDS schema specification to use. If ``None``, uses the bundled
167            default schema.
168
169    Returns:
170        The PyArrow schema of the BIDS dataset index.
171    """
172    adapter = (
173        schema if isinstance(schema, BIDSSchemaAdapter) else load_bids_schema(schema)
174    )
175    entity_schema = entity_arrow_schema(adapter)
176    index_fields = {
177        name: pa.field(name, cfg["dtype"], metadata=cfg["metadata"])
178        for name, cfg in _INDEX_ARROW_FIELDS.items()
179    }
180    fields = [
181        index_fields["dataset"],
182        *entity_schema,
183        index_fields["extra_entities"],
184        index_fields["dataset_name"],
185        index_fields["dataset_type"],
186        index_fields["bids_version"],
187        index_fields["root"],
188        index_fields["path"],
189    ]
190    metadata = {
191        **entity_schema.metadata,
192        b"bids2table_version": version.encode(),
193    }
194    return pa.schema(fields, metadata=metadata)

197def get_column_names(*, schema: SchemaSpec = None) -> type[enum.StrEnum]:
198    """Get an enum of the BIDS index columns.
199
200    Args:
201        schema: BIDS schema specification to use. If ``None``, uses the bundled
202            default schema.
203
204    Returns:
205        A ``str`` enum class whose members are the index column names.
206    """
207    arrow_schema = get_arrow_schema(schema=schema)
208    items = []
209    for f in arrow_schema:
210        name = f.metadata[b"name"].decode()
211        items.append((name, name))
212
213    BIDSColumn = enum.StrEnum("BIDSColumn", items)  # noqa: N806 - class type
214    BIDSColumn.__doc__ = "Enum of BIDS index column names."
215    return BIDSColumn

285def index_dataset(
286    root: str | PathT,
287    include_subjects: str | list[str] | None = None,
288    *,
289    filters: dict[str, str | list[str]] | None = None,
290    schema: SchemaSpec = None,
291) -> pa.Table:
292    """Index a BIDS dataset.
293
294    Args:
295        root: BIDS dataset root directory.
296        include_subjects: Glob pattern or list of patterns for matching subjects to
297            include in the index. .. deprecated:: Use ``filters={'sub': ...}`` instead.
298        filters: Dict mapping entity keys to glob patterns or lists of patterns.
299        schema: BIDS schema specification to use. If ``None``, uses the bundled
300            default schema.
301
302    Returns:
303        An Arrow table index of the BIDS dataset.
304    """
305    root = as_path(root)
306
307    if include_subjects is not None:
308        warnings.warn(
309            "include_subjects is deprecated; use filters={'sub': ...} instead.",
310            DeprecationWarning,
311            stacklevel=2,
312        )
313        if filters is None:
314            filters = {}
315        filters["sub"] = include_subjects
316
317    adapter = load_bids_schema(schema)
318    arrow_schema = get_arrow_schema(schema=adapter)
319
320    dataset, _ = _get_bids_dataset(root)
321    if dataset is None:
322        _logger.warning(f"Path {root} is not a valid BIDS dataset directory.")
323        return pa.Table.from_pylist([], schema=arrow_schema)
324
325    entity_dirs = _resolve_entity_dirs(root, adapter=adapter, filters=filters)
326    entity_dirs = sorted(entity_dirs, key=lambda p: p.name)
327    if len(entity_dirs) == 0:
328        _logger.warning(f"Path {root} contains no matching entity dirs.")
329        return pa.Table.from_pylist([], schema=arrow_schema)
330
331    tables = []
332    file_count = 0
333    for entity_dir in entity_dirs:
334        prefix = entity_dir.name.split("-")[0]
335        _, table = _index_bids_entity_dir(
336            entity_dir, prefix, adapter, arrow_schema, dataset, filters
337        )
338        tables.append(table)
339        file_count += len(table)
340    return pa.concat_tables(tables).combine_chunks()

12def load_bids_metadata(path: str | PathT, *, inherit: bool = True) -> dict[str, Any]:
13    """Load the full JSON sidecar metadata for a BIDS file.
14
15    Sidecar files are loaded according to the inheritance principle in top-down order.
16
17    Args:
18        path: BIDS file path
19        inherit: Load the full metadata according to inheritance. Otherwise, load only
20            the first JSON sidecar found in the bottom-up search.
21
22    Returns:
23        A sidecar metadata dictionary.
24    """
25    path = as_path(path)
26    entities = _cache_parse_bids_entities(path)
27    query = dict(entities, ext=".json")
28
29    metadata = {}
30
31    parent = path.parent
32    if inherit:
33        sidecars = reversed(list(_find_bids_parents(parent, query)))
34    else:
35        sidecars = [next(_find_bids_parents(parent, query))]
36
37    for path in sidecars:
38        try:
39            data = _load_json(path)
40            metadata.update(data)
41        except (json.JSONDecodeError, TypeError):
42            continue
43    return metadata

73def parse_bids_entities(
74    path: str | Path, *, schema: SchemaSpec = None
75) -> dict[str, str]:
76    """Parse entities from BIDS file path.
77
78    Parses all BIDS filename `"{key}-{value}"` entities as well as special entities:
79    datatype, suffix, ext (extension). Does not validate entities or cast to types.
80
81    Args:
82        path: BIDS path to parse.
83        schema: Optional BIDS schema. If ``None``, uses the default schema.
84
85    Returns:
86        A dict mapping BIDS entity keys to values.
87
88    Raises:
89        TypeError: If `schema` is not a valid `SchemaSpec`.
90        ValueError: If the schema contains no directory entities.
91    """
92    if isinstance(path, str):
93        path = Path(path)
94    adapter = load_bids_schema(schema)
95    return _cache_parse_bids_entities(path, adapter)

136def validate_bids_entities(
137    entities: dict[str, Any], *, schema: SchemaSpec = None
138) -> tuple[dict[str, BIDSValue], dict[str, Any]]:
139    """Validate BIDS entities against a BIDS schema.
140
141    Args:
142        entities: dict mapping BIDS keys to unvalidated entities
143        schema: optional `SchemaSpec` (`Namespace | str | PathT | None`).
144            `None` uses the default BIDS schema bundled with bidsschematools.
145
146    Returns:
147        `(valid_entities, extra_entities)` — valid entities cast to the
148        declared type, plus any leftover entries that did not match a
149        known entity or failed validation.
150    """
151    adapter = load_bids_schema(schema)
152    pa_schema = entity_arrow_schema(adapter)
153    return _pyarrow_validate_entities(entities, pa_schema=pa_schema)