Push zunrplorkwkt #70
@@ -5,6 +5,7 @@
|
||||
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentSparseBitMatrix};
|
||||
use obikindex::KmerIndex;
|
||||
use obilayeredmap::layer_dir;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let sparse_root = std::env::args().nth(1).expect("usage: compare_sparse <sparse_root> <dense_root>");
|
||||
@@ -33,8 +34,10 @@ fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
for layer in 0..n_layers {
|
||||
let layer_dir_sparse = part_dir_sparse.join("index").join(format!("layer_{layer}"));
|
||||
let layer_dir_dense = part_dir_dense.join("index").join(format!("layer_{layer}"));
|
||||
let index_dir_sparse = part_dir_sparse.join("index");
|
||||
let index_dir_dense = part_dir_dense.join("index");
|
||||
let layer_dir_sparse = layer_dir(&index_dir_sparse, layer);
|
||||
let layer_dir_dense = layer_dir(&index_dir_dense, layer);
|
||||
|
||||
if !layer_dir_sparse.exists() || !layer_dir_dense.exists() {
|
||||
continue;
|
||||
|
||||
@@ -264,10 +264,8 @@ impl KmerIndex {
|
||||
|
||||
/// Path to the unitigs file for partition `part`, layer `layer`.
|
||||
pub fn layer_unitigs_path(&self, part: usize, layer: usize) -> PathBuf {
|
||||
self.partition.part_dir(part)
|
||||
.join("index")
|
||||
.join(format!("layer_{layer}"))
|
||||
.join("unitigs.bin")
|
||||
let index_dir = self.partition.part_dir(part).join("index");
|
||||
obilayeredmap::layer_dir(&index_dir, layer).join("unitigs.bin")
|
||||
}
|
||||
|
||||
/// Pack all partition matrices into single-file format (presence → .pbmx, counts → .pcmx).
|
||||
@@ -295,7 +293,7 @@ impl KmerIndex {
|
||||
let meta = PartitionMeta::load(&index_dir)
|
||||
.map_err(|e| OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())))?;
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let layer_dir = obilayeredmap::layer_dir(&index_dir, l);
|
||||
let presence_dir = layer_dir.join("presence");
|
||||
let counts_dir = layer_dir.join("counts");
|
||||
if presence_dir.exists() {
|
||||
@@ -335,7 +333,7 @@ impl KmerIndex {
|
||||
Err(e) => return Some(OKIError::Io(std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))),
|
||||
};
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let layer_dir = obilayeredmap::layer_dir(&index_dir, l);
|
||||
let meta_path = layer_dir.join(LayerMeta::FILENAME);
|
||||
if meta_path.exists() { continue; }
|
||||
let unitigs_path = layer_dir.join("unitigs.bin");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use obilayeredmap::{IndexMode, layer::Layer};
|
||||
use obilayeredmap::{layer_dir, IndexMode, layer::Layer};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obisys::{Reporter, Stage, progress_bar};
|
||||
use tracing::info;
|
||||
@@ -72,8 +72,7 @@ fn reindex_partition(index_dir: &Path, target: &IndexMode, block_bits: u8) -> OK
|
||||
}
|
||||
let pm = PartitionMeta::load(index_dir).map_err(olm_to_oki)?;
|
||||
for layer_idx in 0..pm.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{layer_idx}"));
|
||||
reindex_layer(&layer_dir, target, block_bits)?;
|
||||
reindex_layer(&layer_dir(index_dir, layer_idx), target, block_bits)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::path::Path;
|
||||
|
||||
use obicompactvec::{LayerMeta, PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obicompactvec::traits::ColumnWeights;
|
||||
use obilayeredmap::layer_dir;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use rayon::prelude::*;
|
||||
|
||||
@@ -98,7 +99,7 @@ impl KmerIndex {
|
||||
.unwrap_or(0);
|
||||
|
||||
(0..n_layers).fold((0usize, 0u64, 0u64, 0u64), |acc, l| {
|
||||
let lb = layer_bytes(&index_dir.join(format!("layer_{l}")));
|
||||
let lb = layer_bytes(&layer_dir(&index_dir, l));
|
||||
(acc.0 + lb.n_kmers, acc.1 + lb.mphf, acc.2 + lb.evidence, acc.3 + lb.matrix)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,7 +2,7 @@ use std::path::Path;
|
||||
|
||||
use obicompactvec::{PersistentBitVecBuilder, PersistentCompactIntVecBuilder};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obilayeredmap::{IndexMode, OLMError};
|
||||
use obilayeredmap::{layer_dir, IndexMode, OLMError};
|
||||
use obiskio::{SKError, SKResult};
|
||||
|
||||
// ── olm_to_sk ────────────────────────────────────────────────────────────────
|
||||
@@ -27,7 +27,7 @@ pub(crate) fn load_meta(dir: &Path, context: &'static str) -> SKResult<Partition
|
||||
Err(e) if matches!(e, OLMError::Io(ref io_e) if io_e.kind() == std::io::ErrorKind::NotFound) =>
|
||||
{
|
||||
let mut n = 0usize;
|
||||
while dir.join(format!("layer_{n}")).exists() {
|
||||
while layer_dir(dir, n).exists() {
|
||||
n += 1;
|
||||
}
|
||||
let m = PartitionMeta {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::{SKError, SKResult, UnitigFileReader};
|
||||
use obilayeredmap::{IndexMode, MphfLayer, OLMError};
|
||||
use obilayeredmap::{layer_dir, IndexMode, MphfLayer, OLMError};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
|
||||
use crate::filter::{KmerFilter, passes_all};
|
||||
@@ -47,7 +47,7 @@ impl KmerPartition {
|
||||
|
||||
let mut l = 0;
|
||||
loop {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let layer_dir = layer_dir(&index_dir, l);
|
||||
if !layer_dir.exists() { break; }
|
||||
l += 1;
|
||||
let mphf = MphfLayer::open(&layer_dir, &index_mode).map_err(olm_to_sk)?;
|
||||
@@ -127,7 +127,7 @@ impl KmerPartition {
|
||||
|
||||
let mut layer = 0;
|
||||
loop {
|
||||
let layer_dir = index_dir.join(format!("layer_{layer}"));
|
||||
let layer_dir = layer_dir(&index_dir, layer);
|
||||
if !layer_dir.exists() { break; }
|
||||
let mphf = MphfLayer::open(&layer_dir, &index_mode).map_err(olm_to_sk)?;
|
||||
let reader = UnitigFileReader::open_sequential(&layer_dir.join("unitigs.bin"))?;
|
||||
|
||||
@@ -22,7 +22,7 @@ use obipipeline::{
|
||||
use obicompactvec::{PersistentBitMatrixBuilder, PersistentCompactIntMatrixBuilder};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obilayeredmap::{IndexMode, Layer, LayeredMap, MphfOnly};
|
||||
use obilayeredmap::{layer_dir, IndexMode, Layer, LayeredMap, MphfOnly};
|
||||
use obiskio::{SKError, SKResult, UnitigFileReader};
|
||||
|
||||
use crate::common::{ColBuilder, load_meta, olm_to_sk};
|
||||
@@ -226,8 +226,7 @@ impl KmerPartition {
|
||||
// (all slots true — every kmer in those layers belongs to genome_0).
|
||||
if n_dst_genomes == 1 && mode == MergeMode::Presence {
|
||||
for l in 0..n_dst_layers {
|
||||
let layer_dir = dst_index_dir.join(format!("layer_{l}"));
|
||||
Layer::<()>::init_presence_matrix(&layer_dir, dst_map.layer(l).n())
|
||||
Layer::<()>::init_presence_matrix(&layer_dir(&dst_index_dir, l), dst_map.layer(l).n())
|
||||
.map_err(|e| olm_to_sk(e, "merge"))?;
|
||||
}
|
||||
}
|
||||
@@ -248,7 +247,7 @@ impl KmerPartition {
|
||||
}
|
||||
let src_meta = load_meta(&src_index_dir, "merge")?;
|
||||
for l in 0..src_meta.n_layers {
|
||||
let p = src_index_dir.join(format!("layer_{l}")).join("unitigs.bin");
|
||||
let p = layer_dir(&src_index_dir, l).join("unitigs.bin");
|
||||
if p.exists() {
|
||||
unitig_paths.push(p);
|
||||
}
|
||||
@@ -293,7 +292,7 @@ impl KmerPartition {
|
||||
|
||||
// Build new layer from de Bruijn graph if there are new kmers.
|
||||
let new_layer_idx = n_dst_layers;
|
||||
let new_layer_dir = dst_index_dir.join(format!("layer_{new_layer_idx}"));
|
||||
let new_layer_dir = layer_dir(&dst_index_dir, new_layer_idx);
|
||||
|
||||
let n_new = if any_new {
|
||||
debug!("partition {i}: unitig traversal start — {} nodes", g.len());
|
||||
@@ -344,7 +343,7 @@ impl KmerPartition {
|
||||
let mut exist_mbs: Vec<MatrixBuilder> = Vec::with_capacity(n_dst_layers);
|
||||
let mut exist_builders: Vec<Vec<ColBuilder>> = Vec::with_capacity(n_dst_layers);
|
||||
for l in 0..n_dst_layers {
|
||||
let layer_dir = dst_index_dir.join(format!("layer_{l}"));
|
||||
let layer_dir = layer_dir(&dst_index_dir, l);
|
||||
let data_dir = match mode {
|
||||
MergeMode::Presence => layer_dir.join("presence"),
|
||||
MergeMode::Count => layer_dir.join("counts"),
|
||||
@@ -374,7 +373,7 @@ impl KmerPartition {
|
||||
}
|
||||
let src_meta = load_meta(&src_index_dir, "merge")?;
|
||||
for l in 0..src_meta.n_layers {
|
||||
let src_layer_dir = src_index_dir.join(format!("layer_{l}"));
|
||||
let src_layer_dir = layer_dir(&src_index_dir, l);
|
||||
if src_layer_dir.join("unitigs.bin").exists() {
|
||||
pass2_items.push((col_offset, *src_n, src_layer_dir));
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::path::Path;
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix};
|
||||
use obikseq::CanonicalKmer;
|
||||
use obiskio::{SKError, SKResult};
|
||||
use obilayeredmap::{IndexMode, MphfLayer, OLMError};
|
||||
use obilayeredmap::{layer_dir, IndexMode, MphfLayer, OLMError};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
|
||||
use crate::partition::KmerPartition;
|
||||
@@ -182,7 +182,7 @@ impl KmerPartition {
|
||||
|
||||
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_sk)?;
|
||||
let layers: Vec<QueryLayer> = (0..meta.n_layers)
|
||||
.map(|i| QueryLayer::open(&index_dir.join(format!("layer_{i}")), with_counts, &meta.mode))
|
||||
.map(|i| QueryLayer::open(&layer_dir(&index_dir, i), with_counts, &meta.mode))
|
||||
.collect::<SKResult<_>>()?;
|
||||
|
||||
// ── Stage 1: MPHF-only pass, bucket hits by (layer_idx, slot) ────────
|
||||
|
||||
@@ -8,7 +8,7 @@ use obicompactvec::{
|
||||
use obidebruinj::GraphDeBruijn;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obilayeredmap::{IndexMode, MphfLayer};
|
||||
use obilayeredmap::{layer_dir, IndexMode, MphfLayer};
|
||||
use obiskio::{SKError, SKResult, UnitigFileReader};
|
||||
|
||||
use crate::common::{load_meta, olm_to_sk};
|
||||
@@ -112,7 +112,7 @@ fn iter_src_kmers_masked(
|
||||
) -> SKResult<()> {
|
||||
let src_meta = load_meta(src_index_dir, "rebuild")?;
|
||||
for l in 0..src_meta.n_layers {
|
||||
let src_layer_dir = src_index_dir.join(format!("layer_{l}"));
|
||||
let src_layer_dir = layer_dir(src_index_dir, l);
|
||||
let unitigs_path = src_layer_dir.join("unitigs.bin");
|
||||
if !unitigs_path.exists() { continue; }
|
||||
|
||||
@@ -149,7 +149,7 @@ fn iter_src_layers(
|
||||
) -> SKResult<()> {
|
||||
let src_meta = load_meta(src_index_dir, "rebuild")?;
|
||||
for l in 0..src_meta.n_layers {
|
||||
let src_layer_dir = src_index_dir.join(format!("layer_{l}"));
|
||||
let src_layer_dir = layer_dir(src_index_dir, l);
|
||||
let unitigs_path = src_layer_dir.join("unitigs.bin");
|
||||
if !unitigs_path.exists() { continue; }
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ use obicompactvec::{
|
||||
PersistentCompactIntMatrix, PersistentCompactIntMatrixBuilder,
|
||||
};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obilayeredmap::OLMError;
|
||||
use obilayeredmap::{layer_dir, OLMError};
|
||||
use obiskio::{SKError, SKResult};
|
||||
|
||||
use crate::partition::KmerPartition;
|
||||
@@ -165,10 +165,10 @@ impl KmerPartition {
|
||||
let data_subdir = if output_presence { "presence" } else { "counts" };
|
||||
|
||||
for l in 0..src_meta.n_layers {
|
||||
let src_layer_dir = src_index_dir.join(format!("layer_{l}"));
|
||||
let src_layer_dir = layer_dir(&src_index_dir, l);
|
||||
if !src_layer_dir.exists() { continue; }
|
||||
|
||||
let dst_layer_dir = dst_index_dir.join(format!("layer_{l}"));
|
||||
let dst_layer_dir = layer_dir(&dst_index_dir, l);
|
||||
|
||||
let counts_dir = src_layer_dir.join("counts");
|
||||
let presence_dir = src_layer_dir.join("presence");
|
||||
|
||||
@@ -7,7 +7,7 @@ use rayon::prelude::*;
|
||||
use obikpartitionner::KmerPartition;
|
||||
use obipipeline::ThrottleGuard;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::MphfLayer;
|
||||
use obilayeredmap::{layer_dir, MphfLayer};
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obisys::progress_bar;
|
||||
|
||||
@@ -101,8 +101,7 @@ impl SiblingAnnexBuildExt for KmerIndex {
|
||||
|
||||
let mut part_slots: u64 = 0;
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
part_slots += build_layer_sibling_annex(self, &layer_dir, n_parts, l, &cache)?;
|
||||
part_slots += build_layer_sibling_annex(self, &layer_dir(&index_dir, l), n_parts, l, &cache)?;
|
||||
}
|
||||
total_slots += part_slots;
|
||||
pb.inc(1);
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::path::Path;
|
||||
use obicompactvec::{PersistentBitMatrix, PersistentCompactIntMatrix, PersistentSparseBitMatrix};
|
||||
use obikpartitionner::KmerPartition;
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::{Layer, OLMResult};
|
||||
use obilayeredmap::{layer_dir, Layer, OLMResult};
|
||||
use obilayeredmap::meta::{IndexMode, PartitionMeta};
|
||||
use obisys::progress_bar;
|
||||
|
||||
@@ -182,8 +182,7 @@ impl PartitionCache {
|
||||
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
|
||||
let mut mats = Vec::with_capacity(meta.n_layers);
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let Ok(mat) = Mat::open(&layer_dir, &meta.mode, with_counts) else { continue };
|
||||
let Ok(mat) = Mat::open(&layer_dir(&index_dir, l), &meta.mode, with_counts) else { continue };
|
||||
mats.push(mat);
|
||||
}
|
||||
pb.inc(1);
|
||||
|
||||
@@ -54,6 +54,7 @@ use std::sync::atomic::{AtomicU8, Ordering};
|
||||
use rayon::prelude::*;
|
||||
|
||||
use obikseq::CanonicalKmer;
|
||||
use obilayeredmap::layer_dir;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obipipeline::{ThrottleGuard, throttle};
|
||||
|
||||
@@ -111,15 +112,15 @@ pub(crate) fn sibling_layer_dirs(index: &KmerIndex) -> OKIResult<Vec<PathBuf>> {
|
||||
}
|
||||
let meta = PartitionMeta::load(&index_dir).map_err(olm_to_ok)?;
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let annex_path = layer_dir.join(ANNEX_FILE_NAME);
|
||||
let this_layer_dir = layer_dir(&index_dir, l);
|
||||
let annex_path = this_layer_dir.join(ANNEX_FILE_NAME);
|
||||
if !annex_path.exists() {
|
||||
return Err(OKIError::InvalidInput(format!(
|
||||
"no sibling annex at {} — run build_sibling_annex first",
|
||||
annex_path.display()
|
||||
)));
|
||||
}
|
||||
layer_dirs.push(layer_dir);
|
||||
layer_dirs.push(this_layer_dir);
|
||||
}
|
||||
}
|
||||
Ok(layer_dirs)
|
||||
|
||||
@@ -3,6 +3,7 @@ use std::path::Path;
|
||||
|
||||
use obikseq::{CanonicalKmer, Kmer, Sequence};
|
||||
use obilayeredmap::MphfLayer;
|
||||
use obilayeredmap::layer_dir;
|
||||
use obilayeredmap::meta::PartitionMeta;
|
||||
use obisys::Reporter;
|
||||
use tempfile::tempdir;
|
||||
@@ -91,7 +92,7 @@ fn annex_info_for(idx: &KmerIndex, kmer: CanonicalKmer) -> FamilyMask {
|
||||
let index_dir = idx.partition().part_dir(0).join(INDEX_SUBDIR);
|
||||
let meta = PartitionMeta::load(&index_dir).unwrap();
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let layer_dir = layer_dir(&index_dir, l);
|
||||
let mphf = MphfLayer::open(&layer_dir, &meta.mode).unwrap();
|
||||
if let Some(slot) = mphf.find(kmer) {
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).unwrap();
|
||||
@@ -313,7 +314,7 @@ fn sibling_annex_no_empty_masks_after_build() {
|
||||
let index_dir = g1.partition().part_dir(0).join(INDEX_SUBDIR);
|
||||
let meta = PartitionMeta::load(&index_dir).expect("partition meta");
|
||||
for l in 0..meta.n_layers {
|
||||
let layer_dir = index_dir.join(format!("layer_{l}"));
|
||||
let layer_dir = layer_dir(&index_dir, l);
|
||||
let annex = SiblingAnnex::open(&layer_dir.join(ANNEX_FILE_NAME)).expect("annex open");
|
||||
for slot in 0..annex.len() {
|
||||
let mask = annex.get(slot).expect("slot must have an entry");
|
||||
|
||||
Reference in New Issue
Block a user