Unverified Commit aa16ccf5 authored by Anant Sharma's avatar Anant Sharma Committed by GitHub
Browse files

fix: remove default-members in workspace (#6279)


Signed-off-by: default avatarAnant Sharma <anants@nvidia.com>
parent c02cefb5
...@@ -18,21 +18,6 @@ members = [ ...@@ -18,21 +18,6 @@ members = [
"lib/bindings/python/codegen", "lib/bindings/python/codegen",
"lib/config", "lib/config",
] ]
# Exclude certain packages that are slow to build and we don't ship as flagship
# features from default build, but keep them in workspace for convenience.
default-members = [
"lib/llm",
"lib/runtime",
"lib/config",
"lib/tokens",
"lib/mocker",
"lib/memory",
"lib/kvbm-logical",
"lib/async-openai",
"lib/parsers",
"lib/bench",
"lib/bindings/c",
]
resolver = "3" resolver = "3"
[workspace.package] [workspace.package]
......
...@@ -81,18 +81,19 @@ impl PrometheusParser { ...@@ -81,18 +81,19 @@ impl PrometheusParser {
} }
// Apply macro prefix to constants if needed // Apply macro prefix to constants if needed
if is_macro_generated && macro_prefix.is_some() { if is_macro_generated {
let prefix = macro_prefix.as_ref().unwrap(); if let Some(prefix) = macro_prefix.as_ref() {
for constant in &mut constants { for constant in &mut constants {
// Only apply if the constant doesn't already have the prefix // Only apply if the constant doesn't already have the prefix
if constant.name == "PREFIX" { if constant.name == "PREFIX" {
// PREFIX constant should be just the prefix with trailing underscore // PREFIX constant should be just the prefix with trailing underscore
continue; continue;
} }
// Check if value looks like it should have prefix applied // Check if value looks like it should have prefix applied
// (doesn't already start with the prefix) // (doesn't already start with the prefix)
if !constant.value.starts_with(prefix) { if !constant.value.starts_with(prefix) {
constant.value = format!("{}_{}", prefix, constant.value); constant.value = format!("{}_{}", prefix, constant.value);
}
} }
} }
} }
......
...@@ -1134,7 +1134,7 @@ mod tests { ...@@ -1134,7 +1134,7 @@ mod tests {
for i in 0..10 { for i in 0..10 {
indexer indexer
.apply_event(create_store_event(0, i, vec![i as u64], None)) .apply_event(create_store_event(0, i, vec![i], None))
.await; .await;
} }
...@@ -1154,7 +1154,7 @@ mod tests { ...@@ -1154,7 +1154,7 @@ mod tests {
.apply_event(create_store_event( .apply_event(create_store_event(
worker_id, worker_id,
1, 1,
vec![1, 2, worker_id as u64 + 10], vec![1, 2, worker_id + 10],
None, None,
)) ))
.await; .await;
......
...@@ -2400,13 +2400,13 @@ mod tests { ...@@ -2400,13 +2400,13 @@ mod tests {
index.flush().await; index.flush().await;
// Query for full sequence [1, 2, 3, 4, 5] should match all 5 blocks // Query for full sequence [1, 2, 3, 4, 5] should match all 5 blocks
let full_seq: Vec<LocalBlockHash> = (1..=5).map(|i| LocalBlockHash(i)).collect(); let full_seq: Vec<LocalBlockHash> = (1..=5).map(LocalBlockHash).collect();
let scores = index.find_matches(full_seq).await.unwrap(); let scores = index.find_matches(full_seq).await.unwrap();
assert_eq!(scores.scores.len(), 1); assert_eq!(scores.scores.len(), 1);
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 5); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 5);
// Query for just [1, 2, 3] should match 3 blocks // Query for just [1, 2, 3] should match 3 blocks
let prefix_seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let prefix_seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(prefix_seq).await.unwrap(); let scores = index.find_matches(prefix_seq).await.unwrap();
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3);
} }
...@@ -2430,7 +2430,7 @@ mod tests { ...@@ -2430,7 +2430,7 @@ mod tests {
index.flush().await; index.flush().await;
// Query should return all 3 dp_ranks as separate entries // Query should return all 3 dp_ranks as separate entries
let seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq).await.unwrap(); let scores = index.find_matches(seq).await.unwrap();
assert_eq!(scores.scores.len(), 3); assert_eq!(scores.scores.len(), 3);
...@@ -2450,14 +2450,14 @@ mod tests { ...@@ -2450,14 +2450,14 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Verify all 3 blocks match // Verify all 3 blocks match
let seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq.clone()).await.unwrap(); let scores = index.find_matches(seq.clone()).await.unwrap();
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3);
// Remove only the last block (block 3) // Remove only the last block (block 3)
// To do this correctly, we need to compute the seq_hash for block 3 specifically, // To do this correctly, we need to compute the seq_hash for block 3 specifically,
// which requires the full sequence context [1,2,3]. // which requires the full sequence context [1,2,3].
let full_hashes: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let full_hashes: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let seq_hashes = compute_seq_hash_for_block(&full_hashes); let seq_hashes = compute_seq_hash_for_block(&full_hashes);
let block_3_seq_hash = ExternalSequenceBlockHash(seq_hashes[2]); // Last block's hash let block_3_seq_hash = ExternalSequenceBlockHash(seq_hashes[2]); // Last block's hash
...@@ -2480,7 +2480,7 @@ mod tests { ...@@ -2480,7 +2480,7 @@ mod tests {
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 2); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 2);
// Query [1, 2] - should still match 2 blocks // Query [1, 2] - should still match 2 blocks
let partial_seq: Vec<LocalBlockHash> = (1..=2).map(|i| LocalBlockHash(i)).collect(); let partial_seq: Vec<LocalBlockHash> = (1..=2).map(LocalBlockHash).collect();
let scores = index.find_matches(partial_seq).await.unwrap(); let scores = index.find_matches(partial_seq).await.unwrap();
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 2); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 2);
} }
...@@ -2502,7 +2502,7 @@ mod tests { ...@@ -2502,7 +2502,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Worker 0's data should still be there // Worker 0's data should still be there
let seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq).await.unwrap(); let scores = index.find_matches(seq).await.unwrap();
assert_eq!(scores.scores.len(), 1); assert_eq!(scores.scores.len(), 1);
assert!(scores.scores.contains_key(&WorkerWithDpRank::new(0, 0))); assert!(scores.scores.contains_key(&WorkerWithDpRank::new(0, 0)));
...@@ -2522,7 +2522,7 @@ mod tests { ...@@ -2522,7 +2522,7 @@ mod tests {
index.flush().await; index.flush().await;
// Original data should still be there // Original data should still be there
let seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq).await.unwrap(); let scores = index.find_matches(seq).await.unwrap();
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3);
} }
...@@ -2541,7 +2541,7 @@ mod tests { ...@@ -2541,7 +2541,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Verify data is gone // Verify data is gone
let seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq.clone()).await.unwrap(); let scores = index.find_matches(seq.clone()).await.unwrap();
assert!(scores.scores.is_empty()); assert!(scores.scores.is_empty());
...@@ -2572,12 +2572,12 @@ mod tests { ...@@ -2572,12 +2572,12 @@ mod tests {
index.flush().await; index.flush().await;
// Query first sequence // Query first sequence
let seq1: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq1: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq1).await.unwrap(); let scores = index.find_matches(seq1).await.unwrap();
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3);
// Query second sequence // Query second sequence
let seq2: Vec<LocalBlockHash> = (100..=102).map(|i| LocalBlockHash(i)).collect(); let seq2: Vec<LocalBlockHash> = (100..=102).map(LocalBlockHash).collect();
let scores = index.find_matches(seq2).await.unwrap(); let scores = index.find_matches(seq2).await.unwrap();
assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3); assert_eq!(*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), 3);
...@@ -2604,7 +2604,7 @@ mod tests { ...@@ -2604,7 +2604,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Verify both dp_ranks are present // Verify both dp_ranks are present
let seq: Vec<LocalBlockHash> = (1..=3).map(|i| LocalBlockHash(i)).collect(); let seq: Vec<LocalBlockHash> = (1..=3).map(LocalBlockHash).collect();
let scores = index.find_matches(seq.clone()).await.unwrap(); let scores = index.find_matches(seq.clone()).await.unwrap();
assert_eq!(scores.scores.len(), 2); assert_eq!(scores.scores.len(), 2);
...@@ -2647,7 +2647,7 @@ mod tests { ...@@ -2647,7 +2647,7 @@ mod tests {
); );
// Query prefix (first 64 blocks) // Query prefix (first 64 blocks)
let prefix_query: Vec<LocalBlockHash> = (1..=64).map(|i| LocalBlockHash(i)).collect(); let prefix_query: Vec<LocalBlockHash> = (1..=64).map(LocalBlockHash).collect();
let scores = index.find_matches(prefix_query).await.unwrap(); let scores = index.find_matches(prefix_query).await.unwrap();
assert_eq!( assert_eq!(
*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), *scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(),
...@@ -2655,8 +2655,7 @@ mod tests { ...@@ -2655,8 +2655,7 @@ mod tests {
); );
// Query with divergence at position 50 // Query with divergence at position 50
let mut divergent_query: Vec<LocalBlockHash> = let mut divergent_query: Vec<LocalBlockHash> = (1..=100).map(LocalBlockHash).collect();
(1..=100).map(|i| LocalBlockHash(i)).collect();
divergent_query[49] = LocalBlockHash(99999); // Position 49 (0-indexed) diverges divergent_query[49] = LocalBlockHash(99999); // Position 49 (0-indexed) diverges
let scores = index.find_matches(divergent_query).await.unwrap(); let scores = index.find_matches(divergent_query).await.unwrap();
assert_eq!( assert_eq!(
...@@ -2691,7 +2690,7 @@ mod tests { ...@@ -2691,7 +2690,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Query full sequence - should match all 150 blocks // Query full sequence - should match all 150 blocks
let full_query: Vec<LocalBlockHash> = (1..=150).map(|i| LocalBlockHash(i)).collect(); let full_query: Vec<LocalBlockHash> = (1..=150).map(LocalBlockHash).collect();
let scores = index.find_matches(full_query).await.unwrap(); let scores = index.find_matches(full_query).await.unwrap();
assert_eq!(scores.scores.len(), 1); assert_eq!(scores.scores.len(), 1);
assert_eq!( assert_eq!(
...@@ -2700,14 +2699,13 @@ mod tests { ...@@ -2700,14 +2699,13 @@ mod tests {
); );
// Query crossing continuation boundaries // Query crossing continuation boundaries
let cross_boundary_query: Vec<LocalBlockHash> = let cross_boundary_query: Vec<LocalBlockHash> = (45..=105).map(LocalBlockHash).collect();
(45..=105).map(|i| LocalBlockHash(i)).collect();
let scores = index.find_matches(cross_boundary_query).await.unwrap(); let scores = index.find_matches(cross_boundary_query).await.unwrap();
// Query starts at block 45, but stored sequence starts at 1, so this won't match // Query starts at block 45, but stored sequence starts at 1, so this won't match
// because the sequence hash at position 0 of our query (block 45) won't match // because the sequence hash at position 0 of our query (block 45) won't match
// the stored sequence hash at position 0 (block 1) // the stored sequence hash at position 0 (block 1)
assert!( assert!(
scores.scores.is_empty() || scores.scores.get(&WorkerWithDpRank::new(0, 0)).is_none() scores.scores.is_empty() || !scores.scores.contains_key(&WorkerWithDpRank::new(0, 0))
); );
} }
...@@ -2737,7 +2735,7 @@ mod tests { ...@@ -2737,7 +2735,7 @@ mod tests {
index.flush().await; index.flush().await;
// Query common prefix - both workers should match // Query common prefix - both workers should match
let prefix_query: Vec<LocalBlockHash> = (1..=30).map(|i| LocalBlockHash(i)).collect(); let prefix_query: Vec<LocalBlockHash> = (1..=30).map(LocalBlockHash).collect();
let scores = index.find_matches(prefix_query).await.unwrap(); let scores = index.find_matches(prefix_query).await.unwrap();
assert_eq!(scores.scores.len(), 2); assert_eq!(scores.scores.len(), 2);
assert_eq!( assert_eq!(
...@@ -2750,7 +2748,7 @@ mod tests { ...@@ -2750,7 +2748,7 @@ mod tests {
); );
// Query branch A path - only worker 0 should match fully // Query branch A path - only worker 0 should match fully
let branch_a_query: Vec<LocalBlockHash> = (1..=60).map(|i| LocalBlockHash(i)).collect(); let branch_a_query: Vec<LocalBlockHash> = (1..=60).map(LocalBlockHash).collect();
let scores = index.find_matches(branch_a_query).await.unwrap(); let scores = index.find_matches(branch_a_query).await.unwrap();
assert_eq!( assert_eq!(
*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), *scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(),
...@@ -2782,7 +2780,7 @@ mod tests { ...@@ -2782,7 +2780,7 @@ mod tests {
); );
// Remove blocks 80-100 (the tail) // Remove blocks 80-100 (the tail)
let tail_hashes: Vec<LocalBlockHash> = (1..=100).map(|i| LocalBlockHash(i)).collect(); let tail_hashes: Vec<LocalBlockHash> = (1..=100).map(LocalBlockHash).collect();
let seq_hashes = compute_seq_hash_for_block(&tail_hashes); let seq_hashes = compute_seq_hash_for_block(&tail_hashes);
let remove_hashes: Vec<ExternalSequenceBlockHash> = seq_hashes[79..100] let remove_hashes: Vec<ExternalSequenceBlockHash> = seq_hashes[79..100]
.iter() .iter()
...@@ -2835,7 +2833,7 @@ mod tests { ...@@ -2835,7 +2833,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Query for 60 blocks - workers 0,1 match 60, worker 2 matches 50, worker 3 matches 25 // Query for 60 blocks - workers 0,1 match 60, worker 2 matches 50, worker 3 matches 25
let query_60: Vec<LocalBlockHash> = (1..=60).map(|i| LocalBlockHash(i)).collect(); let query_60: Vec<LocalBlockHash> = (1..=60).map(LocalBlockHash).collect();
let scores = index.find_matches(query_60).await.unwrap(); let scores = index.find_matches(query_60).await.unwrap();
assert_eq!(scores.scores.len(), 4); assert_eq!(scores.scores.len(), 4);
assert_eq!( assert_eq!(
...@@ -2962,7 +2960,7 @@ mod tests { ...@@ -2962,7 +2960,7 @@ mod tests {
// Test divergence exactly at jump boundaries (position 31, 32, 33, 63, 64, 65) // Test divergence exactly at jump boundaries (position 31, 32, 33, 63, 64, 65)
for diverge_pos in [31usize, 32, 33, 63, 64, 65, 95, 96, 97] { for diverge_pos in [31usize, 32, 33, 63, 64, 65, 95, 96, 97] {
let mut query: Vec<LocalBlockHash> = (1..=128).map(|i| LocalBlockHash(i)).collect(); let mut query: Vec<LocalBlockHash> = (1..=128).map(LocalBlockHash).collect();
query[diverge_pos] = LocalBlockHash(99999); query[diverge_pos] = LocalBlockHash(99999);
let scores = index.find_matches(query).await.unwrap(); let scores = index.find_matches(query).await.unwrap();
...@@ -3008,7 +3006,7 @@ mod tests { ...@@ -3008,7 +3006,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Query full sequence // Query full sequence
let full_query: Vec<LocalBlockHash> = (1..=200).map(|i| LocalBlockHash(i)).collect(); let full_query: Vec<LocalBlockHash> = (1..=200).map(LocalBlockHash).collect();
let scores = index.find_matches(full_query).await.unwrap(); let scores = index.find_matches(full_query).await.unwrap();
assert_eq!( assert_eq!(
*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), *scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(),
...@@ -3016,7 +3014,7 @@ mod tests { ...@@ -3016,7 +3014,7 @@ mod tests {
); );
// Query partial prefix crossing multiple chunk boundaries // Query partial prefix crossing multiple chunk boundaries
let partial_query: Vec<LocalBlockHash> = (1..=75).map(|i| LocalBlockHash(i)).collect(); let partial_query: Vec<LocalBlockHash> = (1..=75).map(LocalBlockHash).collect();
let scores = index.find_matches(partial_query).await.unwrap(); let scores = index.find_matches(partial_query).await.unwrap();
assert_eq!( assert_eq!(
*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), *scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(),
...@@ -3155,7 +3153,7 @@ mod tests { ...@@ -3155,7 +3153,7 @@ mod tests {
tokio::time::sleep(Duration::from_millis(100)).await; tokio::time::sleep(Duration::from_millis(100)).await;
// Query for 100 blocks - each worker should match their stored length // Query for 100 blocks - each worker should match their stored length
let query: Vec<LocalBlockHash> = (1..=100).map(|i| LocalBlockHash(i)).collect(); let query: Vec<LocalBlockHash> = (1..=100).map(LocalBlockHash).collect();
let scores = index.find_matches(query).await.unwrap(); let scores = index.find_matches(query).await.unwrap();
assert_eq!( assert_eq!(
...@@ -3201,7 +3199,7 @@ mod tests { ...@@ -3201,7 +3199,7 @@ mod tests {
); );
// Partial match (first 500) // Partial match (first 500)
let partial_query: Vec<LocalBlockHash> = (1..=500).map(|i| LocalBlockHash(i)).collect(); let partial_query: Vec<LocalBlockHash> = (1..=500).map(LocalBlockHash).collect();
let scores = index.find_matches(partial_query).await.unwrap(); let scores = index.find_matches(partial_query).await.unwrap();
assert_eq!( assert_eq!(
*scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(), *scores.scores.get(&WorkerWithDpRank::new(0, 0)).unwrap(),
...@@ -3209,7 +3207,7 @@ mod tests { ...@@ -3209,7 +3207,7 @@ mod tests {
); );
// Divergence in the middle // Divergence in the middle
let mut mid_diverge: Vec<LocalBlockHash> = (1..=1000).map(|i| LocalBlockHash(i)).collect(); let mut mid_diverge: Vec<LocalBlockHash> = (1..=1000).map(LocalBlockHash).collect();
mid_diverge[499] = LocalBlockHash(99999); mid_diverge[499] = LocalBlockHash(99999);
let scores = index.find_matches(mid_diverge).await.unwrap(); let scores = index.find_matches(mid_diverge).await.unwrap();
assert_eq!( assert_eq!(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment