Anndata Data Structure

pip install "anndata>=0.10"
# Full ecosystem
pip install anndata scanpy zarr

import anndata as ad
import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix

counts = csr_matrix(np.random.poisson(0.5, (500, 2000)).astype(np.float32))
obs = pd.DataFrame({"cell_type": np.random.choice(["T", "B", "NK"], 500)},
                    index=[f"cell_{i}" for i in range(500)])
var = pd.DataFrame(index=[f"ENSG{i:05d}" for i in range(2000)])
adata = ad.AnnData(X=counts, obs=obs, var=var)
adata.layers["raw_counts"] = counts.copy()
adata.write_h5ad("example.h5ad", compression="gzip")
print(f"Created: {adata.n_obs} cells x {adata.n_vars} genes")
# Created: 500 cells x 2000 genes

import anndata as ad
import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix

# Minimal: just a matrix
adata_min = ad.AnnData(X=np.random.rand(100, 50).astype(np.float32))
print(f"Minimal: {adata_min.shape}")  # (100, 50)

# Full: sparse matrix + obs/var metadata
n_obs, n_vars = 300, 1000
X = csr_matrix(np.random.poisson(1, (n_obs, n_vars)).astype(np.float32))
obs = pd.DataFrame({"cell_type": np.random.choice(["T", "B", "Mono"], n_obs),
                     "batch": np.repeat(["ctrl", "stim"], n_obs // 2)},
                    index=[f"cell_{i}" for i in range(n_obs)])
var = pd.DataFrame({"gene_symbol": [f"Gene_{i}" for i in range(n_vars)],
                     "mt": [i < 13 for i in range(n_vars)]},
                    index=[f"ENSG{i:05d}" for i in range(n_vars)])
adata = ad.AnnData(X=X, obs=obs, var=var)
print(f"Full: {adata.shape}, obs cols: {list(adata.obs.columns)}")
# Full: (300, 1000), obs cols: ['cell_type', 'batch']

# From a pandas DataFrame (rows=obs, columns=vars)
df = pd.DataFrame(np.random.rand(50, 20),
                  index=[f"sample_{i}" for i in range(50)],
                  columns=[f"feature_{i}" for i in range(20)])
adata_df = ad.AnnData(df)
print(f"From DataFrame: {adata_df.shape}")  # (50, 20)

import anndata as ad

# H5AD (native format, recommended for most use cases)
adata = ad.read_h5ad("data.h5ad")
adata.write_h5ad("output.h5ad", compression="gzip")  # gzip: smaller files

# 10X Genomics formats
adata_10x = ad.read_10x_h5("filtered_feature_bc_matrix.h5")
# adata_mtx = ad.read_10x_mtx("filtered_feature_bc_matrix/")

# Zarr format (cloud-friendly, parallel I/O)
adata.write_zarr("output.zarr")
adata_zarr = ad.read_zarr("output.zarr")

# Other formats
# adata = ad.read_csv("expression.csv")
# adata = ad.read_loom("data.loom")

print(f"Loaded: {adata.n_obs} obs x {adata.n_vars} vars")

import anndata as ad

# Backed mode: lazy loading for files larger than RAM
adata_backed = ad.read_h5ad("large_data.h5ad", backed="r")  # read-only
print(f"Backed: {adata_backed.n_obs} obs, isbacked={adata_backed.isbacked}")

# Filter on metadata (no data loaded), then load subset into memory
subset = adata_backed[adata_backed.obs["tissue"] == "brain"].to_memory()
print(f"Loaded subset: {subset.n_obs} cells")

# Read-write backed mode: adata_rw = ad.read_h5ad("data.h5ad", backed="r+")
# Format conversion: ad.read_loom("data.loom").write_h5ad("out.h5ad", compression="gzip")

import anndata as ad

adata = ad.read_h5ad("data.h5ad")

# Boolean mask (most common)
t_cells = adata[adata.obs["cell_type"] == "T_cell"]
print(f"T cells: {t_cells.n_obs}, is_view: {t_cells.is_view}")  # is_view: True

# Integer index / name-based / combined axis
first_100 = adata[:100, :500]
selected = adata[["cell_0", "cell_1"], ["ENSG00000", "ENSG00001"]]

# Combined metadata conditions
high_quality = adata[
    (adata.obs["n_genes"] > 200) & (adata.obs["pct_mito"] < 0.2)
]
print(f"QC filter: {high_quality.n_obs} / {adata.n_obs} cells")

# Views vs copies: subsetting returns a view (lightweight, shares data)
# .copy() creates an independent object (REQUIRED before modification)
independent = adata[adata.obs["batch"] == "ctrl"].copy()
print(f"Is view: {independent.is_view}")  # False

import anndata as ad
import numpy as np
from scipy.sparse import csr_matrix

adata = ad.read_h5ad("data.h5ad")

# Layers: alternative representations of X (same shape as X)
adata.layers["raw_counts"] = adata.X.copy()
adata.layers["normalized"] = adata.X.copy()
print(f"Layers: {list(adata.layers.keys())}")
# Layers: ['raw_counts', 'normalized']

# Embeddings in obsm (n_obs x n_components)
adata.obsm["X_pca"] = np.random.randn(adata.n_obs, 50).astype(np.float32)
adata.obsm["X_umap"] = np.random.randn(adata.n_obs, 2).astype(np.float32)
print(f"obsm keys: {list(adata.obsm.keys())}")

# Variable loadings in varm (n_vars x n_components)
adata.varm["PCs"] = np.random.randn(adata.n_vars, 50).astype(np.float32)

# Pairwise graphs in obsp (n_obs x n_obs, sparse)
adata.obsp["connectivities"] = csr_matrix(
    np.random.rand(adata.n_obs, adata.n_obs) > 0.99)
adata.obsp["distances"] = adata.obsp["connectivities"].copy()

# Unstructured metadata in uns (arbitrary dict)
adata.uns["experiment"] = {"date": "2024-06-01", "protocol": "10x_v3"}
adata.uns["neighbors"] = {"params": {"n_neighbors": 15, "method": "umap"}}
adata.uns["cell_type_colors"] = ["#1f77b4", "#ff7f0e", "#2ca02c"]
print(f"uns keys: {list(adata.uns.keys())}")

import anndata as ad
import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix

# Create sample datasets
def make_adata(n, genes, batch_name):
    X = csr_matrix(np.random.poisson(1, (n, len(genes))).astype(np.float32))
    obs = pd.DataFrame({"sample": batch_name}, index=[f"{batch_name}_{i}" for i in range(n)])
    return ad.AnnData(X=X, obs=obs, var=pd.DataFrame(index=genes))

shared = [f"Gene_{i}" for i in range(100)]
adata1 = make_adata(200, shared + ["GeneA"], "batch1")
adata2 = make_adata(300, shared + ["GeneB"], "batch2")

# Along observations (axis=0): stack cells
combined = ad.concat(
    [adata1, adata2], axis=0, join="inner",
    label="batch", keys=["B1", "B2"], merge="same",
)
print(f"Inner join: {combined.n_obs} cells, {combined.n_vars} genes")
# Inner join: 500 cells, 100 genes

# Outer join: keeps all genes, fills missing with NaN/0
combined_outer = ad.concat([adata1, adata2], join="outer")
print(f"Outer join: {combined_outer.n_vars} genes")  # 102 genes

# Along variables (axis=1): multi-modal
n = 100
obs = pd.DataFrame(index=[f"cell_{i}" for i in range(n)])
rna = ad.AnnData(X=csr_matrix(np.random.poisson(1, (n, 500)).astype(np.float32)),
                 obs=obs, var=pd.DataFrame(index=[f"RNA_{i}" for i in range(500)]))
protein = ad.AnnData(X=csr_matrix(np.random.rand(n, 50).astype(np.float32)),
                     obs=obs, var=pd.DataFrame(index=[f"ADT_{i}" for i in range(50)]))
multimodal = ad.concat([rna, protein], axis=1)
print(f"Multimodal: {multimodal.shape}")  # (100, 550)

# Lazy concatenation for very large datasets (no data copying)
from anndata.experimental import AnnCollection

collection = AnnCollection(
    {"batch1": adata1, "batch2": adata2},
    join_obs="inner",
)
print(f"Lazy collection: {collection.n_obs} total obs")
# On-disk concat (writes directly to disk without loading all into memory)
# ad.experimental.concat_on_disk({"b1": "batch1.h5ad", "b2": "batch2.h5ad"}, "combined.h5ad")

import anndata as ad
import numpy as np
from scipy.sparse import csr_matrix, issparse

adata = ad.read_h5ad("data.h5ad")

# Type conversions
adata.strings_to_categoricals()  # string cols -> categorical (saves memory)
if not issparse(adata.X):
    adata.X = csr_matrix(adata.X)  # dense -> sparse
dense_X = adata.X.toarray() if issparse(adata.X) else adata.X  # sparse -> dense

# Adding/removing metadata columns
adata.obs["log_counts"] = np.log1p(np.array(adata.X.sum(axis=1)).flatten())
adata.var["mean_expr"] = np.array(adata.X.mean(axis=0)).flatten()
del adata.obs["unwanted_column"]  # remove

# Renaming observations/variables/categories
adata.obs_names_make_unique()  # add suffixes to duplicate names
adata.var_names_make_unique()
adata.obs["cell_type"] = adata.obs["cell_type"].cat.rename_categories(
    {"T": "T_cell", "B": "B_cell"})

# Quality control filtering (always .copy() after subsetting)
adata.obs["n_genes"] = np.array((adata.X > 0).sum(axis=1)).flatten()
mito_mask = adata.var_names.str.startswith("MT-")
adata.obs["pct_mito"] = (np.array(adata[:, mito_mask].X.sum(axis=1)).flatten()
                          / np.array(adata.X.sum(axis=1)).flatten())
adata_qc = adata[(adata.obs["n_genes"] > 200) & (adata.obs["pct_mito"] < 0.2)].copy()
print(f"After QC: {adata_qc.n_obs} / {adata.n_obs} cells")

view = adata[adata.obs["cell_type"] == "T_cell"]
print(f"is_view: {view.is_view}")        # True -- shares memory
independent = view.copy()
print(f"is_view: {independent.is_view}")  # False -- independent

import anndata as ad
import numpy as np
from scipy.sparse import issparse

# 1. Load and QC filter (see Core API 6 for metric computation details)
adata = ad.read_h5ad("raw_counts.h5ad")
adata.obs["n_genes"] = np.array((adata.X > 0).sum(axis=1)).flatten()
adata.obs["total_counts"] = np.array(adata.X.sum(axis=1)).flatten()
mito = adata.var_names.str.startswith("MT-")
adata.obs["pct_mito"] = (np.array(adata[:, mito].X.sum(axis=1)).flatten()
                          / np.array(adata.X.sum(axis=1)).flatten())
adata = adata[(adata.obs["n_genes"].between(200, 5000)) &
              (adata.obs["pct_mito"] < 0.2)].copy()
adata = adata[:, np.array((adata.X > 0).sum(axis=0)).flatten() >= 3].copy()

# 2. Store raw counts, then normalize (total-count + log1p)
adata.layers["counts"] = adata.X.copy()
totals = np.array(adata.X.sum(axis=1)).flatten()
if issparse(adata.X):
    adata.X = np.log1p(adata.X.multiply(1.0 / totals[:, None]).toarray() * 1e4)
else:
    adata.X = np.log1p(adata.X / totals[:, None] * 1e4)

# 3. Save
adata.strings_to_categoricals()
adata.write_h5ad("processed.h5ad", compression="gzip")
print(f"Saved: {adata.n_obs} cells x {adata.n_vars} genes, layers: {list(adata.layers.keys())}")

import anndata as ad
from pathlib import Path

# 1. Load all batches
batches = {}
for h5 in sorted(Path("batches/").glob("*.h5ad")):
    batches[h5.stem] = ad.read_h5ad(str(h5))
    print(f"  {h5.stem}: {batches[h5.stem].n_obs} cells")

# 2. Harmonize genes and concatenate
shared = set.intersection(*[set(a.var_names) for a in batches.values()])
batches = {k: v[:, list(shared)].copy() for k, v in batches.items()}
combined = ad.concat(batches, label="batch", join="inner", merge="same")

# 3. Clean up and save
combined.obs_names_make_unique()
combined.strings_to_categoricals()
combined.write_h5ad("combined_batches.h5ad", compression="gzip")
print(f"Combined: {combined.n_obs} cells x {combined.n_vars} genes, "
      f"{combined.obs['batch'].nunique()} batches")