obikmer/src/obicompactvec/examples/compare_sparse_dense_count.rs

//! Diagnostic: build a `PersistentSparseCompactIntMatrix` from a real dense
//! `PersistentCompactIntMatrix` (batched `build_from_dense`), verify the
//! result is cell-for-cell identical on an actual count matrix, and report
//! the on-disk compaction ratio, value-distribution stats, and effective
//! bits/value.
//!
//! Usage: cargo run --release --example compare_sparse_dense_count -p obicompactvec -- <layer_dir>
//! (layer_dir is the directory containing `counts/matrix.pcmx` or
//! `counts/col_*.pciv`, e.g.
//! `benchmark/global_index_count/partitions/part_00003/index/layer_1`)

use std::error::Error;
use std::path::Path;
use std::time::Instant;

use obicompactvec::{PersistentCompactIntMatrix, PersistentSparseCompactIntMatrix, PersistentSparseCompactIntMatrixBuilder};

fn main() -> Result<(), Box<dyn Error>> {
    let layer_dir = std::env::args().nth(1).expect("usage: compare_sparse_dense_count <layer_dir>");
    let layer_dir = Path::new(&layer_dir);

    let t0 = Instant::now();
    let dense = PersistentCompactIntMatrix::open(layer_dir)?;
    println!("dense ouverte en {:?} ({} lignes x {} colonnes)", t0.elapsed(), dense.n(), dense.n_cols());

    let out_dir = tempfile::tempdir()?;

    let t0 = Instant::now();
    let sparse = PersistentSparseCompactIntMatrixBuilder::build_from_dense(&dense, out_dir.path())?.finish()?;
    println!("build_from_dense: {:?}", t0.elapsed());

    compare(&dense, &sparse)?;
    content_stats(&dense);
    compaction_stats(layer_dir, out_dir.path(), &dense, &sparse)?;

    Ok(())
}

fn compare(dense: &PersistentCompactIntMatrix, sparse: &PersistentSparseCompactIntMatrix) -> Result<(), Box<dyn Error>> {
    let n = dense.n();
    let n_cols = dense.n_cols();
    assert_eq!(n, sparse.n(), "n mismatch");
    assert_eq!(n_cols, sparse.n_cols(), "n_cols mismatch");

    let mut dense_row = vec![0u32; n_cols];
    let mut sparse_row = vec![0u32; n_cols];
    let mut total_cells = 0usize;
    let mut mismatched_cells = 0usize;
    let mut first_mismatch = None;

    let t0 = Instant::now();
    for slot in 0..n {
        dense.fill_row(slot, &mut dense_row);
        sparse.fill_row(slot, &mut sparse_row);

        for c in 0..n_cols {
            total_cells += 1;
            if dense_row[c] != sparse_row[c] {
                mismatched_cells += 1;
                if first_mismatch.is_none() {
                    first_mismatch = Some((slot, c, dense_row[c], sparse_row[c]));
                }
            }
        }
    }
    println!("comparaison case-a-case: {:?} ({total_cells} cellules)", t0.elapsed());

    if let Some((slot, col, d, s)) = first_mismatch {
        eprintln!("PREMIER MISMATCH: slot={slot} col={col} dense={d} sparse={s}");
        println!("MISMATCHES: {mismatched_cells} cellules differentes sur {total_cells}");
        return Err("dense et sparse divergent".into());
    }
    println!("OK: toutes les {total_cells} cellules sont identiques entre dense et sparse.");
    Ok(())
}

/// Content stats computed straight from the dense matrix (source of truth
/// for what's actually stored) — sparsity, singleton-row fraction, and the
/// value-magnitude buckets this crate's overflow encoding is tuned around
/// (< 127, < 255, >= 255).
fn content_stats(dense: &PersistentCompactIntMatrix) {
    let n = dense.n();
    let n_cols = dense.n_cols();
    let total_cells = n as u64 * n_cols as u64;

    let mut nonzero_cells = 0u64;
    let mut singleton_rows = 0u64;
    let mut sum: u128 = 0;
    let mut max_value = 0u32;
    let mut under_127 = 0u64;
    let mut under_255 = 0u64;
    let mut overflow = 0u64;

    let mut row = vec![0u32; n_cols];
    for slot in 0..n {
        dense.fill_row(slot, &mut row);
        let mut row_nonzero = 0u32;
        for &v in &row {
            if v == 0 {
                continue;
            }
            row_nonzero += 1;
            nonzero_cells += 1;
            sum += v as u128;
            max_value = max_value.max(v);
            if v < 127 {
                under_127 += 1;
            }
            if v < 255 {
                under_255 += 1;
            } else {
                overflow += 1;
            }
        }
        if row_nonzero == 1 {
            singleton_rows += 1;
        }
    }

    println!("\n--- stats de contenu (source: dense) ---");
    println!("cellules totales:      {total_cells}");
    println!(
        "cellules non-nulles:   {nonzero_cells} ({:.3}% du total)",
        100.0 * nonzero_cells as f64 / total_cells as f64
    );
    println!(
        "lignes singleton:      {singleton_rows} / {n} ({:.3}%)",
        100.0 * singleton_rows as f64 / n as f64
    );
    if nonzero_cells > 0 {
        println!("valeur moyenne (non-nulles): {:.2}", sum as f64 / nonzero_cells as f64);
        println!("valeur max:                  {max_value}");
        println!(
            "valeurs < 127:  {under_127} ({:.3}% des non-nulles)",
            100.0 * under_127 as f64 / nonzero_cells as f64
        );
        println!(
            "valeurs < 255:  {under_255} ({:.3}% des non-nulles)",
            100.0 * under_255 as f64 / nonzero_cells as f64
        );
        println!(
            "valeurs >= 255 (overflow): {overflow} ({:.3}% des non-nulles)",
            100.0 * overflow as f64 / nonzero_cells as f64
        );
    }
}

/// Directory size in bytes — sums every regular file, one level deep
/// (matches both the dense `counts/` layout and the sparse builder's flat
/// output directory, neither of which nests further).
fn dir_size(dir: &Path) -> std::io::Result<u64> {
    let mut total = 0u64;
    for entry in std::fs::read_dir(dir)? {
        let entry = entry?;
        if entry.file_type()?.is_file() {
            total += entry.metadata()?.len();
        }
    }
    Ok(total)
}

fn compaction_stats(
    dense_layer_dir: &Path,
    sparse_dir: &Path,
    dense: &PersistentCompactIntMatrix,
    _sparse: &PersistentSparseCompactIntMatrix,
) -> Result<(), Box<dyn Error>> {
    let dense_size = dir_size(&dense_layer_dir.join("counts"))?;
    let sparse_size = dir_size(sparse_dir)?;

    let n = dense.n();
    let n_cols = dense.n_cols();
    let nonzero_cells: u64 = {
        let mut row = vec![0u32; n_cols];
        let mut count = 0u64;
        for slot in 0..n {
            dense.fill_row(slot, &mut row);
            count += row.iter().filter(|&&v| v != 0).count() as u64;
        }
        count
    };

    println!("\n--- compaction ---");
    println!("taille dense (counts/):  {dense_size} octets");
    println!("taille sparse:           {sparse_size} octets");
    println!(
        "ratio sparse/dense:      {:.4} ({:.2}% de la taille dense)",
        sparse_size as f64 / dense_size as f64,
        100.0 * sparse_size as f64 / dense_size as f64
    );
    if nonzero_cells > 0 {
        println!(
            "bits/valeur (dense, sur cellules non-nulles):  {:.3}",
            dense_size as f64 * 8.0 / nonzero_cells as f64
        );
        println!(
            "bits/valeur (sparse, sur cellules non-nulles): {:.3}",
            sparse_size as f64 * 8.0 / nonzero_cells as f64
        );
    }
    let total_cells = n as u64 * n_cols as u64;
    println!(
        "bits/valeur (dense,  sur toutes les cellules): {:.3}",
        dense_size as f64 * 8.0 / total_cells as f64
    );
    println!(
        "bits/valeur (sparse, sur toutes les cellules): {:.3}",
        sparse_size as f64 * 8.0 / total_cells as f64
    );

    Ok(())
}