Out-of-memory Processing¶

"""Combine filtered peak tables from many samples without loading them into RAM."""

from pathlib import Path
from tempfile import TemporaryDirectory

import emzed

with TemporaryDirectory() as tmp:
    folder = Path(tmp)

    # Pretend we already have peak tables from several samples on disk,
    # produced by earlier per-sample processing.
    sample_paths = []
    for i in range(3):
        t = emzed.Table.create_table(
            ["mz", "rt", "area"],
            [emzed.MzType, emzed.RtType, float],
            rows=[
                [150.0 + i, 30.0 + i, 1e5 * (i + 1)],
                [151.0 + i, 31.0 + i, 0.5e5 * (i + 1)],
            ],
        )
        path = folder / f"sample_{i}.table"
        t.save(path)
        sample_paths.append(path)

    # Open each sample on disk and keep only high-quality peaks.
    filtered = []
    for path in sample_paths:
        t = emzed.Table.open(path)
        # keep_view=True keeps the filter as an on-disk view;
        # no rows are loaded into RAM.
        t = t.filter(t.area > 0.8e5, keep_view=True)
        filtered.append(t)

    # Combine the filtered views into one on-disk table.
    combined = emzed.Table.stack_tables(
        filtered,
        path=folder / "combined.table",
        overwrite=True,
    )

    print("combined is on disk:", not combined.is_in_memory())
    print(combined)