diff --git a/crates/integration-tests/tests/production_rollup_workloads.rs b/crates/integration-tests/tests/production_rollup_workloads.rs index e00b2ba7c..e22d027fc 100644 --- a/crates/integration-tests/tests/production_rollup_workloads.rs +++ b/crates/integration-tests/tests/production_rollup_workloads.rs @@ -292,3 +292,59 @@ fn independent_kll_cost( let result = asap_plan_selection::select_plan(&[plan], entries, data, models).unwrap(); result.costs[&result.selected].cost_per_second.unwrap() } + +// Exercise multi-input operators through the production route under more than +// three heterogeneous query schedules, not just the local pricing kernel. +#[test] +fn production_binary_queries_use_joint_prices_with_mixed_schedules() { + use asap_types::workload::{DemandEstimate, Rate}; + let accuracy = AccuracyTarget::exact(); + let queries = ["m + n", "m + m", "m or n", "m * n", "m - n"]; + let roots = queries + .iter() + .enumerate() + .map(|(id, query)| { + let root = lower_promql(query, accuracy.clone()).unwrap(); + ( + id, + QueryRoot::Operator(promql_rows::with_series_identity(&root).unwrap()), + ) + }) + .collect(); + let mut entries: Vec<_> = [59_000, 61_000, 67_000, 71_000, 73_000] + .into_iter() + .map(|interval| entry(&accuracy, interval)) + .collect(); + entries[3].recurrence = + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(0.137), + observed_at: None, + valid_for: None, + })); + entries[4].recurrence = QueryRecurrence::Unknown; + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival":{"ingesting":{"rows_per_second":16.0}}, + "sources":{"m":{"rows_held":80,"series_count":8,"sample_interval":1000}, + "n":{"rows_held":80,"series_count":8,"sample_interval":1000}} + })) + .unwrap(); + let run = production(roots, &entries, &data); + let selection = run + .stage3_plan_selection + .as_ref() + .expect("validated binary workload"); + let report = selection.search.candidate_connections.as_ref().unwrap(); + println!("mixed binary workload report={report:?}"); + assert!( + report.joint_price_solve_calls > 0 && report.direct_cost_validated_plans > 0, + "{report:?}" + ); + let accepted = &selection.accepted[&selection.selected]; + assert_eq!(accepted.len(), queries.len()); + assert!(accepted + .iter() + .all(|item| !item.accepted_regardless_of_guarantee)); + assert!(selection.costs[&selection.selected] + .cost_per_second + .is_some_and(|cost| cost.is_finite() && cost >= 0.0)); +} diff --git a/crates/plan-selection/src/cost/execution_demand.rs b/crates/plan-selection/src/cost/execution_demand.rs index 5b912765f..fc930fcc6 100644 --- a/crates/plan-selection/src/cost/execution_demand.rs +++ b/crates/plan-selection/src/cost/execution_demand.rs @@ -120,6 +120,33 @@ pub(crate) fn evaluation_rate<'r>( Ok(repeated + estimated + invocations as f64 / non_repeating_query_horizon_seconds) } +/// Number of offsets common to nonempty arithmetic progressions +/// `{k * step | k * step <= span}`. A checked LCM beyond the shortest span +/// means only offset zero is shared. Kept in costing so both evaluator and +/// MILP demand atoms use the same integer arithmetic without layer inversion. +pub(crate) fn offset_intersection_count(geometries: &[(u64, u64)]) -> i128 { + fn gcd(mut a: u64, mut b: u64) -> u64 { + while b != 0 { + let remainder = a % b; + a = b; + b = remainder; + } + a + } + let Some(span) = geometries.iter().map(|(_, span)| *span).min() else { + return 0; + }; + let mut lcm = 1u64; + for &(step, _) in geometries { + let step = step.max(1); + match (lcm / gcd(lcm, step)).checked_mul(step) { + Some(next) if next <= span => lcm = next, + _ => return 1, + } + } + i128::from(span / lcm) + 1 +} + /// The distinct instants one request evaluates at, for steps in ms each with /// the longest span in ms at that step: the union of the offsets /// `k · step_ms ≤ span_ms` back from the request's end. One step needs no @@ -129,6 +156,30 @@ fn distinct_instants_per_request(longest_span_ms_by_step_ms: &BTreeMap let (&step_ms, &span_ms) = longest_span_ms_by_step_ms.iter().next().expect("one step"); return span_ms / step_ms.max(1) + 1; } + // Inclusion-exclusion is bounded by 4095 intersections for up to twelve + // distinct steps, regardless of the number of evaluations in each range. + if longest_span_ms_by_step_ms.len() <= 12 { + let geometries: Vec<_> = longest_span_ms_by_step_ms + .iter() + .map(|(&step, &span)| (step, span)) + .collect(); + let mut total = 0i128; + for mask in 1usize..(1usize << geometries.len()) { + let intersection: Vec<_> = geometries + .iter() + .enumerate() + .filter(|(bit, _)| mask & (1 << bit) != 0) + .map(|(_, geometry)| *geometry) + .collect(); + let count = offset_intersection_count(&intersection); + if mask.count_ones() % 2 == 1 { + total += count; + } else { + total -= count; + } + } + return u64::try_from(total).expect("distinct evaluation offset count fits u64"); + } let mut offsets_ms = BTreeSet::new(); for (&step_ms, &span_ms) in longest_span_ms_by_step_ms { offsets_ms.extend((0..=span_ms / step_ms.max(1)).map(|k| k * step_ms)); @@ -373,6 +424,65 @@ mod tests { ); } + #[test] + fn billion_offset_mixed_ranges_are_counted_arithmetically() { + let recurrence = fixed(1000, None); + let rate = evaluation_rate( + [ + ( + &recurrence, + Some(RangeEvaluation { + span_ms: 1_000_000_000, + step_ms: 2, + }), + ), + ( + &recurrence, + Some(RangeEvaluation { + span_ms: 1_000_000_000, + step_ms: 3, + }), + ), + ], + 60.0, + ) + .unwrap(); + assert_eq!(rate, 666_666_668.0); + } + + #[test] + fn arithmetic_union_matches_small_explicit_offset_sets() { + for left_step in 1..=7 { + for right_step in 1..=7 { + for left_span in [0, 5, 17, 30] { + for right_span in [0, 9, 23, 30] { + let mut ranges = BTreeMap::new(); + for (step, span) in + [(left_step, left_span), (right_step, right_span), (11, 29)] + { + let longest = ranges.entry(step).or_insert(0u64); + *longest = (*longest).max(span); + } + let mut oracle = BTreeSet::new(); + for (&step, &span) in &ranges { + oracle.extend((0..=span / step).map(|k| k * step)); + } + assert_eq!( + distinct_instants_per_request(&ranges), + oracle.len() as u64, + "{ranges:?}" + ); + } + } + } + } + let large = 1u64 << 63; + assert_eq!( + offset_intersection_count(&[(large - 1, large), (large - 3, large)]), + 1 + ); + } + // Invalid fixed intervals and estimated rates still produce the evaluator's error. #[test] fn invalid_recurrence_is_rejected() { diff --git a/crates/plan-selection/src/search/costing/direct_price_options_tests.rs b/crates/plan-selection/src/search/costing/direct_price_options_tests.rs index 70bca171b..2370ea24b 100644 --- a/crates/plan-selection/src/search/costing/direct_price_options_tests.rs +++ b/crates/plan-selection/src/search/costing/direct_price_options_tests.rs @@ -76,7 +76,7 @@ fn differing_supplier_statistics_keep_both_price_interfaces() { .collect(), }; let roots = vec![QueryRoot::Operator(project)]; - let entries = vec![QueryWorkloadEntry { + let entries = [QueryWorkloadEntry { query: Query("m".into()), requirements: QueryRequirements { accuracy: AccuracyTarget::exact(), @@ -118,21 +118,22 @@ fn differing_supplier_statistics_keep_both_price_interfaces() { .iter() .find(|option| { option - .compatible_suppliers + .compatible_input_tuples .iter() + .flatten() .any(|binding| binding.producer == producer) }) .expect("missing compatible supplier price option"); - assert_eq!(option.compatible_suppliers.len(), 1); - assert_eq!(option.compatible_suppliers[0].price_option, 0); + assert_eq!(option.compatible_input_tuples.len(), 1); + assert_eq!(option.compatible_input_tuples[0][0].price_option, 0); } let low = reader_options .iter() - .find(|option| option.compatible_suppliers[0].producer == PhysicalCandidateId(0)) + .find(|option| option.compatible_input_tuples[0][0].producer == PhysicalCandidateId(0)) .unwrap(); let high = reader_options .iter() - .find(|option| option.compatible_suppliers[0].producer == PhysicalCandidateId(1)) + .find(|option| option.compatible_input_tuples[0][0].producer == PhysicalCandidateId(1)) .unwrap(); assert!( high.profile.work_seconds > low.profile.work_seconds, @@ -299,7 +300,7 @@ fn downstream_prices_follow_the_selected_supplier_interface() { }); configured.inventory.graph.query_roots[0].compatible_candidates = vec![PhysicalCandidateId(3)]; let roots = vec![QueryRoot::Operator(downstream)]; - let entries = vec![QueryWorkloadEntry { + let entries = [QueryWorkloadEntry { query: Query("m".into()), requirements: QueryRequirements { accuracy: AccuracyTarget::exact(), @@ -336,8 +337,8 @@ fn downstream_prices_follow_the_selected_supplier_interface() { ); let mut referenced = Vec::new(); for option in &extracted.candidates[3] { - assert_eq!(option.compatible_suppliers.len(), 1); - let binding = &option.compatible_suppliers[0]; + assert_eq!(option.compatible_input_tuples.len(), 1); + let binding = &option.compatible_input_tuples[0][0]; assert_eq!(binding.producer, PhysicalCandidateId(2)); referenced.push(binding.price_option); let supplier = &extracted.candidates[2][binding.price_option]; @@ -378,7 +379,7 @@ fn downstream_prices_follow_the_selected_supplier_interface() { assert_eq!(cyclic.candidates[3].len(), 2); assert!(cyclic.candidates[2] .iter() - .flat_map(|option| &option.compatible_suppliers) + .flat_map(|option| option.compatible_input_tuples.iter().flatten()) .all(|binding| binding.producer != PhysicalCandidateId(2))); let chosen = solve(&configured, &cyclic); assert!( @@ -635,8 +636,9 @@ fn supplier_worker_buckets_remain_joint_choices_and_match_evaluator() { .iter() .find(|option| { option - .compatible_suppliers + .compatible_input_tuples .iter() + .flatten() .any(|binding| binding.producer == raw_supplier) }) .unwrap(); @@ -644,8 +646,9 @@ fn supplier_worker_buckets_remain_joint_choices_and_match_evaluator() { .iter() .find(|option| { option - .compatible_suppliers + .compatible_input_tuples .iter() + .flatten() .any(|binding| binding.producer == projected_supplier) }) .unwrap(); @@ -698,3 +701,152 @@ fn supplier_worker_buckets_remain_joint_choices_and_match_evaluator() { ); } } + +#[test] +fn mixed_schedule_joint_prices_match_the_real_evaluator() { + let scan = |metric| { + OperatorNode::reachable(&lower_promql(metric, AccuracyTarget::exact())) + .into_iter() + .find(|node| matches!(node.non_asap(), Some(NonASAPOp::Scan { .. }))) + .unwrap() + }; + let m = scan("m"); + let n = scan("n"); + let project = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Project { + cols: m + .schema + .fields + .iter() + .enumerate() + .map(|(index, field)| ProjectItem { + alias: Some(field.name.clone()), + expr: ScalarExpr::Column(index), + }) + .collect(), + qualifier: None, + child: m.clone(), + })) + .unwrap(); + let mut configured = CandidateConfigurationInventory { + inventory: CandidateInventory { + graph: CandidateDependencyGraph { + candidates: (0..3) + .map(|id| PhysicalCandidate { + group: MemoGroupId(id as u32), + local_cost: 0.0, + producer_requirements: if id == 2 { + vec![ProducerRequirement { + compatible_producers: vec![ + PhysicalCandidateId(0), + PhysicalCandidateId(1), + ] + .into(), + }] + } else { + vec![] + }, + }) + .collect(), + query_roots: vec![QueryRootCandidates { + compatible_candidates: vec![PhysicalCandidateId(2)], + }], + ..Default::default() + }, + templates: vec![Some(m), Some(n), Some(project.clone())], + group_candidates: vec![], + }, + configurations: (0..3) + .map(|id| CandidateConfiguration { + source_candidate: PhysicalCandidateId(id), + materialization: Materialization::NotMaterialized, + timing: ExecutionTiming::QueryTime, + }) + .collect(), + }; + let roots = vec![QueryRoot::Operator(project); 5]; + let entries = [QueryWorkloadEntry { + query: Query("m".into()), + requirements: QueryRequirements { + accuracy: AccuracyTarget::exact(), + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + range_evaluation: None, + }]; + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival": {"ingesting": {"rows_per_second": 100.0}}, + "sources": {"m": {"rows_held":10,"series_count":1},"n":{"rows_held":100,"series_count":1}} + })) + .unwrap(); + use asap_types::workload::{DemandEstimate, Rate, TimestampMs}; + let mut entries = vec![entries[0].clone(); 5]; + entries[0].recurrence = + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(59_000))); + entries[1].recurrence = + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(61_000))); + entries[2].recurrence = + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(0.137), + observed_at: None, + valid_for: None, + })); + entries[3].recurrence = QueryRecurrence::Repeated(RepeatedDemand::Scheduled(vec![ + TimestampMs(10), + TimestampMs(20), + ])); + entries[4].recurrence = QueryRecurrence::OneTime { + invocations: 3, + execute_at: None, + }; + configured.inventory.graph.query_roots = + vec![configured.inventory.graph.query_roots[0].clone(); 5]; + // All roots use the original source; this parity check has no synthetic + // equivalence between different source names. + configured.inventory.graph.candidates[2].producer_requirements[0].compatible_producers = + vec![PhysicalCandidateId(0)].into(); + let context = CandidatePlanningContext { + deadline: None, + refine_materialization: false, + original_roots: &roots, + entries: &entries, + data: &data, + models: crate::PlanningModels::builtin(), + calibration: crate::CalibrationFit::default(), + }; + let profile = extract_price_options(&configured, &context) + .unwrap() + .unwrap(); + assert!(profile.schedules.is_some()); + let proposal = + crate::search::milp::direct_price_options::solve_joint_price_proposal_with_metrics( + &configured.inventory.graph, + &[], + None, + &mut Default::default(), + &profile, + ) + .unwrap() + .unwrap(); + let mut rejections = crate::search::refinement::planner::CandidateRejectionSummary::default(); + let evaluated = crate::search::refinement::planner::evaluate_configured_candidate( + &configured, + &context, + &proposal.selected, + 0, + &mut rejections, + &mut Vec::new(), + ) + .unwrap() + .unwrap_or_else(|| panic!("{rejections:?}")); + let actual = evaluated.0.costs[&evaluated.0.selected] + .cost_per_second + .unwrap(); + let modeled = proposal.direct_modeled_cost.unwrap(); + assert!( + (actual - modeled).abs() <= 1e-9 * actual.max(1e-15), + "mixed schedules: {modeled} vs {actual}" + ); +} diff --git a/crates/plan-selection/src/search/costing/direct_profile.rs b/crates/plan-selection/src/search/costing/direct_profile.rs index f431af283..428235f63 100644 --- a/crates/plan-selection/src/search/costing/direct_profile.rs +++ b/crates/plan-selection/src/search/costing/direct_profile.rs @@ -44,6 +44,8 @@ fn supported_node(node: &asap_types::ir::OperatorNode) -> bool { | NonASAPOp::Filter { .. } | NonASAPOp::Project { .. } | NonASAPOp::Aggregate { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::Join { .. } ) ) || matches!( node.asap(), @@ -57,9 +59,9 @@ fn supported_node(node: &asap_types::ir::OperatorNode) -> bool { ) } -/// Identical fixed schedules or up to three heterogeneous schedules, no range requests and one operator -/// root per query are supported. Every returned profile is priced at one -/// execution; the MILP supplies demand, held lifetime, retention and overhead. +/// Fixed-interface compatibility adapter. Local profiles are priced at one +/// execution; the MILP supplies schedule demand, lifetime and shared charges. +/// Joint extraction additionally carries input tuples and retention tiers. pub(in crate::search) fn extract( configured: &CandidateConfigurationInventory, context: &CandidatePlanningContext<'_>, @@ -120,15 +122,6 @@ fn prepare_profile( .entries .iter() .any(|entry| entry.range_evaluation.is_some()); - if has_range - && (!equal_fixed - || context - .entries - .iter() - .any(|entry| entry.range_evaluation != first.range_evaluation)) - { - return Ok(None); - } let rates: Vec<_> = context .entries .iter() @@ -140,10 +133,15 @@ fn prepare_profile( .map_err(|error| DirectProfileError::Invalid(error.to_string())) }) .collect::>()?; - if rates.iter().any(|rate| !rate.is_finite() || *rate <= 0.0) { + if rates.iter().any(|rate| !rate.is_finite() || *rate < 0.0) { return Ok(None); } - let rate = rates[0]; + let rate = rates + .iter() + .copied() + .filter(|rate| *rate > 0.0) + .reduce(f64::max) + .unwrap_or(1.0); let request_rate: f64 = context .entries .iter() @@ -157,61 +155,18 @@ fn prepare_profile( .collect::, _>>()? .iter() .sum(); - let schedules = if equal_fixed { + let schedules = if equal_fixed && !has_range { None - } else if context.entries.len() > 3 { - use crate::search::milp::direct_cost::FixedQuerySchedule; - let fixed: Option> = context - .entries - .iter() - .map(|entry| match &entry.recurrence { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval)) => { - Some(FixedQuerySchedule { - interval_ms: interval.0, - phase_ms: None, - }) - } - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval, - evaluation_phase, - }) => Some(FixedQuerySchedule { - interval_ms: interval.0, - phase_ms: Some(evaluation_phase.0), - }), - _ => None, - }) - .collect(); - let Some(fixed) = fixed else { return Ok(None) }; - match QueryScheduleRates::from_fixed_schedules(fixed) { - Ok(schedule) => Some(schedule), - Err( - crate::search::candidates::graph::CandidateSelectionError::UnsupportedDirectCost(_), - ) => return Ok(None), - Err(error) => return Err(DirectProfileError::Invalid(error.to_string())), - } } else { - let mut materialized_subset_rates = Vec::with_capacity(1 << context.entries.len()); - for mask in 0..(1usize << context.entries.len()) { - materialized_subset_rates.push( - crate::cost::execution_demand::evaluation_rate( - context - .entries - .iter() - .enumerate() - .filter(|(query, _)| mask & (1 << query) != 0) - .map(|(_, entry)| (&entry.recurrence, None)), - settings.non_repeating_query_horizon_seconds, - ) - .map_err(|error| DirectProfileError::Invalid(error.to_string()))?, - ); - } - Some(QueryScheduleRates { - compact_fixed: None, - per_query_rates: rates, - materialized_subset_rates, - }) + let general = crate::search::milp::direct_cost::QueryScheduleRates::new( + context.entries, + settings.non_repeating_query_horizon_seconds, + context.deadline, + ) + .map_err(|error| DirectProfileError::Unsupported(error.to_string()))?; + Some(general) }; - let delta = match failure_budget(context) { + let delta = match failure_budget(context)? { Some(delta) => delta, None => return Ok(None), }; @@ -233,7 +188,7 @@ fn prepare_profile( && match configuration.timing { ExecutionTiming::QueryTime => { matches!(configuration.materialization, Materialization::NotMaterialized | Materialization::MatAtQuery(Medium::Memory)) - || (!has_range && configuration.materialization == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) + || (configuration.materialization == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) && matches!(node.asap(), Some(ASAPOp::PaneChain { .. })) && context.models.deployment_capabilities.query_time_results_retained && (0..context.entries.len()).any(|reader| asap_physical_asap_optimization::summary_materialization_eligibility(node, &[reader], context.entries, context.data, context.models.deployment_capabilities).is_ok_and(|eligibility| eligibility.may_keep_across_evaluations))) @@ -257,10 +212,7 @@ fn prepare_profile( } }, } - && configured.inventory.graph.candidates[id] - .producer_requirements - .len() - <= 1 + }) .map(|node| { let mut position = 0; @@ -369,6 +321,23 @@ fn extract_scan_class( context: &CandidatePlanningContext<'_>, prefer_latest_pane: bool, ) -> Result, DirectProfileError> { + // The fixed adapter has no selected-reader retention tiers. The joint + // option extractor carries those decisions instead. + if context + .entries + .iter() + .any(|entry| entry.range_evaluation != context.entries[0].range_evaluation) + || context + .entries + .iter() + .any(|entry| entry.range_evaluation.is_some()) + && configured.configurations.iter().any(|c| { + c.materialization + == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) + }) + { + return Ok(None); + } let Some(prepared) = prepare_profile(configured, context, prefer_latest_pane)? else { return Ok(None); }; @@ -381,6 +350,16 @@ fn extract_scan_class( request_rate, .. } = prepared; + let mut nodes = nodes; + for (id, node) in nodes.iter_mut().enumerate() { + if configured.inventory.graph.candidates[id] + .producer_requirements + .len() + > 1 + { + *node = None; + } + } let count = nodes.len(); let settings = context.models.plan_selection; let CostModel::UsageOverTime(prices) = &settings.cost_model else { @@ -418,6 +397,7 @@ fn extract_scan_class( let supported: Vec<_> = cache.iter().map(Option::is_some).collect(); let zero = || NodeCostProfile { + per_request: false, execution_cost: 0.0, work_seconds: 0.0, compaction_seconds: 0.0, @@ -478,6 +458,8 @@ fn profile_for_priced( }) .transpose()?; Ok(NodeCostProfile { + per_request: node.materialization + == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), execution_cost: priced.cpu_ops_per_second * prices.cost_per_cpu_op + priced.scanned_bytes_per_second * prices.cost_per_scanned_byte, ingestion: node.output_state.timing == ExecutionTiming::IngestionTime, @@ -493,6 +475,13 @@ fn profile_for_priced( }) } +struct PriceInputs { + suppliers: Vec, + upstream_work: WorkBounds, + upstream_sketches: usize, + range_span_ms: Option, +} + #[allow(clippy::too_many_arguments)] fn price<'a>( id: usize, @@ -510,7 +499,7 @@ fn price<'a>( worker_counts: &mut [Option], compaction_seconds: &mut [f64], delta: Option, - supplier_override: Option, + input_override: Option<&PriceInputs>, ) -> Result<(), DirectProfileError> { check_deadline(context.deadline)?; if visited[id] || visiting[id] { @@ -522,27 +511,26 @@ fn price<'a>( }; visiting[id] = true; let requirements = &configured.inventory.graph.candidates[id].producer_requirements; - let options: Vec<_> = supplier_override.map_or_else( + let options: Vec> = input_override.map_or_else( || { requirements.first().map_or_else( - || vec![None], + || vec![vec![]], |input| { input .compatible_producers .iter() - .copied() - .map(Some) + .map(|id| vec![*id]) .collect() }, ) }, - |supplier| vec![Some(supplier)], + |inputs| vec![inputs.suppliers.clone()], ); - for supplier in options { + 'bindings: for input_tuple in options { let mut views = Vec::new(); - if let Some(supplier) = supplier { + for &supplier in &input_tuple { if visiting[supplier.0] { - continue; + continue 'bindings; } price( supplier.0, @@ -563,7 +551,7 @@ fn price<'a>( None, )?; let Some(priced) = &cache[supplier.0] else { - continue; + continue 'bindings; }; let producer = nodes[supplier.0].as_ref().unwrap(); if producer.output_state.timing == ExecutionTiming::QueryTime @@ -573,7 +561,7 @@ fn price<'a>( ) && lookbacks[supplier.0] != scan_consumer_lookback(node) { - continue; + continue 'bindings; } let consumer_ingests = node.output_state.timing == ExecutionTiming::IngestionTime; @@ -584,7 +572,7 @@ fn price<'a>( && producer_ingests && producer.materialization != Materialization::MatAtIngestion(Medium::Memory)) { - continue; + continue 'bindings; } views.push(ProducerOutput { payload: &producer.payload, @@ -595,6 +583,7 @@ fn price<'a>( unchanged_columns: priced.unchanged_columns.clone(), }); } + let supplier = input_tuple.first().copied(); if let asap_types::ir::Operator::ASAP( ASAPOp::SummaryAgg { family: FieldDataType::Sketch(kind, _), @@ -640,22 +629,24 @@ fn price<'a>( } _ => None, }; - let sketches = supplier.map_or(0, |supplier| sketch_counts[supplier.0]) - + usize::from(matches!( - &node.payload, - asap_types::ir::Operator::ASAP( - ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(..), - .. - } | ASAPOp::PaneChain { - family: FieldDataType::Sketch(..), - .. - } - ) - )); + let sketches = input_override.map_or_else( + || supplier.map_or(0, |supplier| sketch_counts[supplier.0]), + |inputs| inputs.upstream_sketches, + ) + usize::from(matches!( + &node.payload, + asap_types::ir::Operator::ASAP( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + } | ASAPOp::PaneChain { + family: FieldDataType::Sketch(..), + .. + } + ) + )); // Failure-budget sizing splits delta across approximate inputs. This // slice admits at most one sketch build along each unary output path. - if sketches > 1 { + if sketches > 1 && input_override.is_none() { continue; } let (retained_panes, ingestion_window) = match &node.payload { @@ -665,11 +656,15 @@ fn price<'a>( .. }) => { let window = pane_width_ms.checked_mul(u64::from(*pane_count)); - let span = context - .entries - .first() - .and_then(|entry| entry.range_evaluation) - .map_or(0, |range| range.span_ms); + let span = input_override + .and_then(|inputs| inputs.range_span_ms) + .unwrap_or_else(|| { + context + .entries + .first() + .and_then(|entry| entry.range_evaluation) + .map_or(0, |range| range.span_ms) + }); let Some(retention) = window.and_then(|window| window.checked_add(span)) else { continue; }; @@ -691,9 +686,14 @@ fn price<'a>( ) else { continue; }; - let prior = supplier.map_or(WorkBounds::default(), |supplier| { - closure_work_ms[supplier.0] - }); + let prior = input_override.map_or_else( + || { + supplier.map_or(WorkBounds::default(), |supplier| { + closure_work_ms[supplier.0] + }) + }, + |inputs| inputs.upstream_work, + ); let closure_work = WorkBounds { minimum: (prior.minimum + priced.work_ms).next_down().max(0.0), maximum: (prior.maximum + priced.work_ms).next_up(), @@ -771,53 +771,111 @@ fn price<'a>( /// A conservative delta valid for every selected unary output path, including /// rollups whose finer grouping the current evaluator uses for its row bound. /// Unknown population bounds reject this direct proposal, not the full search. -fn failure_budget(context: &CandidatePlanningContext<'_>) -> Option> { - let population = || { - let mut sources = HashSet::new(); - for root in context.original_roots { - let QueryRoot::Operator(root) = root else { - return None; - }; - for node in asap_types::ir::OperatorNode::reachable(root) { - if let Some(NonASAPOp::Scan { source, .. }) = node.non_asap() { - sources.insert(crate::source_name(source).to_string()); +fn failure_budget( + context: &CandidatePlanningContext<'_>, +) -> Result>, DirectProfileError> { + check_deadline(context.deadline)?; + let result = (|| { + let population = || { + let mut sources = HashSet::new(); + for root in context.original_roots { + let QueryRoot::Operator(root) = root else { + return None; + }; + for node in asap_types::ir::OperatorNode::reachable(root) { + if let Some(NonASAPOp::Scan { source, .. }) = node.non_asap() { + sources.insert(crate::source_name(source).to_string()); + } } } + let counts: Option> = sources + .iter() + .map(|source| { + context + .data + .sources + .get(source)? + .series_count + .filter(|count| *count > 0) + }) + .collect(); + let maximum_source = counts?.into_iter().max()?; + fn bound( + node: &std::rc::Rc, + maximum_source: u64, + deadline: Option, + memo: &mut std::collections::HashMap, + ) -> Option { + if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) { + return None; + } + let identity = std::rc::Rc::as_ptr(node) as usize; + if let Some(bound) = memo.get(&identity) { + return Some(*bound); + } + let mut children = Vec::new(); + node.operator.map_children(|child| { + children.push(bound(child, maximum_source, deadline, memo)) + }); + let children: Vec<_> = children.into_iter().collect::>()?; + let value = if children.is_empty() { + maximum_source + } else if children.len() == 1 { + children[0] + } else { + // Branching operators can concatenate rows or pair inputs. + // Both union and cross-product bounds cover supported joins, + // set operations and vector matching; sharing does not reduce + // the number of logical input positions in this calculation. + let sum = children + .iter() + .try_fold(0u64, |sum, child| sum.checked_add(*child))?; + let product = children + .iter() + .try_fold(1u64, |product, child| product.checked_mul(*child))?; + sum.max(product) + }; + memo.insert(identity, value); + Some(value) + } + let mut memo = std::collections::HashMap::new(); + context + .original_roots + .iter() + .map(|root| match root { + QueryRoot::Operator(node) => { + bound(node, maximum_source, context.deadline, &mut memo) + } + _ => None, + }) + .collect::>>()? + .into_iter() + .max() + }; + let mut delta: Option = None; + for entry in context.entries { + let target = &entry.requirements.accuracy; + if target.is_exact() { + continue; + } + let requested = target + .failure_probability + .filter(|delta| delta.is_finite() && *delta > 0.0)?; + let values = match target.accuracy_scope { + AccuracyScope::WholeResult => population()?, + scope => scope.max_values_covered(None)?, + } + .max(1); + let required = requested / values as f64; + if !required.is_finite() || required <= 0.0 { + return None; + } + delta = Some(delta.map_or(required, |prior| prior.min(required))); } - let counts: Option> = sources - .iter() - .map(|source| { - context - .data - .sources - .get(source)? - .series_count - .filter(|count| *count > 0) - }) - .collect(); - counts?.into_iter().max() - }; - let mut delta: Option = None; - for entry in context.entries { - let target = &entry.requirements.accuracy; - if target.is_exact() { - continue; - } - let requested = target - .failure_probability - .filter(|delta| delta.is_finite() && *delta > 0.0)?; - let values = match target.accuracy_scope { - AccuracyScope::WholeResult => population()?, - scope => scope.max_values_covered(None)?, - } - .max(1); - let required = requested / values as f64; - if !required.is_finite() || required <= 0.0 { - return None; - } - delta = Some(delta.map_or(required, |prior| prior.min(required))); - } - Some(delta) + Some(delta) + })(); + check_deadline(context.deadline)?; + Ok(result) } fn requires_source_sizing(node: &asap_types::ir::OperatorNode) -> bool { @@ -2094,17 +2152,11 @@ mod tests { .schedules .as_ref() .unwrap() - .materialized_subset_rates[3], + .union_rate([true, true].into_iter()), if phases { 2.0 } else { 3.0 } ); } else { - assert!(extracted - .profile - .schedules - .as_ref() - .unwrap() - .compact_fixed - .is_some()); + assert!(extracted.profile.schedules.is_some()); } let proposal = crate::search::milp::model::solve_direct_cost_proposal_with_metrics( &configured.inventory.graph, @@ -2261,3 +2313,11 @@ pub(in crate::search) use supplier_price_options::extract_price_options; #[cfg(test)] #[path = "direct_price_options_tests.rs"] mod price_options_tests; + +#[cfg(test)] +#[path = "multi_input_price_tests.rs"] +mod multi_input_price_tests; + +#[cfg(test)] +#[path = "range_price_options_tests.rs"] +mod range_price_options_tests; diff --git a/crates/plan-selection/src/search/costing/multi_input_price_tests.rs b/crates/plan-selection/src/search/costing/multi_input_price_tests.rs new file mode 100644 index 000000000..768897b10 --- /dev/null +++ b/crates/plan-selection/src/search/costing/multi_input_price_tests.rs @@ -0,0 +1,292 @@ +//! End-to-end pricing uses the evaluator's binary operators and shared DAGs. +use super::*; +use crate::search::candidates::configurations::CandidateConfiguration; +use crate::search::candidates::graph::*; +use crate::search::candidates::inventory::CandidateInventory; +use asap_logical_asap_optimization::{memo::MemoGroupId, test_support::lower_promql}; +use asap_types::ir::{properties::AccuracyTarget, OperatorNode}; +use asap_types::workload::{ + DataWorkload, Predictability, Query, QueryRequirements, QueryWorkloadEntry, RepetitionInterval, +}; +use std::rc::Rc; + +fn append( + node: &Rc, + inventory: &mut CandidateConfigurationInventory, +) -> PhysicalCandidateId { + // Exact fixture expressions can be interned safely; repeated operands must + // share one producer while keeping their two ordered input positions. + if let Some(id) = inventory.inventory.templates.iter().position(|known| { + known + .as_ref() + .is_some_and(|known| known.as_ref() == node.as_ref()) + }) { + return PhysicalCandidateId(id); + } + let mut inputs = Vec::new(); + node.operator + .map_children(|child| inputs.push(append(child, inventory))); + let id = PhysicalCandidateId(inventory.configurations.len()); + inventory + .inventory + .graph + .candidates + .push(PhysicalCandidate { + group: MemoGroupId(id.0 as u32), + local_cost: 0.0, + producer_requirements: inputs + .into_iter() + .map(|producer| ProducerRequirement { + compatible_producers: vec![producer].into(), + }) + .collect(), + }); + inventory.inventory.templates.push(Some(node.clone())); + inventory.configurations.push(CandidateConfiguration { + source_candidate: id, + materialization: Materialization::NotMaterialized, + timing: ExecutionTiming::QueryTime, + }); + id +} + +#[test] +fn binary_prices_match_evaluator_with_distinct_and_shared_inputs() { + for query in ["m + n", "m + m", "m or n"] { + let logical = lower_promql(query, AccuracyTarget::exact()); + let mut configured = CandidateConfigurationInventory { + inventory: CandidateInventory { + graph: Default::default(), + templates: vec![], + group_candidates: vec![], + }, + configurations: vec![], + }; + let root = append(&logical, &mut configured); + configured + .inventory + .graph + .query_roots + .push(QueryRootCandidates { + compatible_candidates: vec![root], + }); + assert!(configured + .inventory + .graph + .candidates + .iter() + .any(|candidate| candidate.producer_requirements.len() == 2)); + let roots = vec![QueryRoot::Operator(logical)]; + let entries = vec![QueryWorkloadEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy: AccuracyTarget::exact(), + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( + RepetitionInterval(60_000), + )), + range_evaluation: None, + }]; + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival":{"ingesting":{"rows_per_second":100.0}}, + "sources":{"m":{"rows_held":10,"series_count":1},"n":{"rows_held":100,"series_count":1}} + })) + .unwrap(); + let context = CandidatePlanningContext { + deadline: None, + refine_materialization: false, + original_roots: &roots, + entries: &entries, + data: &data, + models: crate::PlanningModels::builtin(), + calibration: crate::CalibrationFit::default(), + }; + let profile = extract_price_options(&configured, &context) + .unwrap() + .unwrap(); + assert!( + !profile.candidates[root.0].is_empty(), + "unsupported binary fixture {query}" + ); + let proposal = + crate::search::milp::direct_price_options::solve_joint_price_proposal_with_metrics( + &configured.inventory.graph, + &[], + None, + &mut Default::default(), + &profile, + ) + .unwrap() + .unwrap(); + let mut rejections = + crate::search::refinement::planner::CandidateRejectionSummary::default(); + let evaluated = crate::search::refinement::planner::evaluate_configured_candidate( + &configured, + &context, + &proposal.selected, + 0, + &mut rejections, + &mut Vec::new(), + ) + .unwrap() + .unwrap_or_else(|| panic!("{query}: {rejections:?}")); + let actual = evaluated.0.costs[&evaluated.0.selected] + .cost_per_second + .unwrap(); + let direct = proposal.direct_modeled_cost.unwrap(); + assert!( + (actual - direct).abs() <= 1e-9 * actual.max(1e-15), + "{query}: actual={actual}, direct={direct}" + ); + if query == "m + m" { + let binary = configured + .inventory + .graph + .candidates + .iter() + .enumerate() + .find(|(_, candidate)| candidate.producer_requirements.len() == 2) + .unwrap() + .0; + let edges: Vec<_> = proposal + .selected + .connections + .iter() + .filter(|edge| edge.consumer.0 == binary) + .collect(); + assert_eq!(edges.len(), 2); + assert_eq!(edges[0].producer, edges[1].producer); + assert_ne!(edges[0].requirement, edges[1].requirement); + } + } +} + +// Two independently approximate inputs must retain their divided confidence +// budgets; pricing may not silently resize either state after selection. +#[test] +fn two_sketch_binary_prices_match_evaluator_after_budget_sizing() { + use asap_logical_asap_optimization::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, + }; + use asap_types::ir::properties::{AccuracyMetric, AccuracyScope}; + use asap_types::ir::schema::SketchAlgorithm; + let query = "quantile_over_time(0.99, m[5m]) / quantile_over_time(0.5, n[5m])"; + let target = AccuracyTarget::max_error(0.05) + .accuracy_metric(AccuracyMetric::Relative) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + let planned = AccuracyTarget::max_error(0.01) + .accuracy_metric(AccuracyMetric::Relative) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + let original = lower_promql(query, planned); + let generated = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(original.clone()))]) + .unwrap(); + let choices: Vec<_> = generated + .targets + .iter() + .map(|target| { + target + .choices + .candidates + .iter() + .position(|choice| { + matches!( + choice, + ComputationCandidate::Summary(SketchAlgorithm::DDSketch) + ) + }) + .unwrap() + }) + .collect(); + assert_eq!(choices.len(), 2); + let roots: Vec<_> = compose_logical_candidate(&generated, &choices) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival":{"ingesting":{"rows_per_second":100.0}}, + "sources":{"m":{"rows_held":3000,"series_count":1,"values_non_negative":true}, + "n":{"rows_held":3000,"series_count":1,"values_non_negative":true}} + })) + .unwrap(); + let sized = crate::resize_for_failure_probability(&roots, std::slice::from_ref(&target), &data) + .unwrap(); + let QueryRoot::Operator(root) = &sized[0] else { + panic!("operator root") + }; + let mut configured = CandidateConfigurationInventory { + inventory: CandidateInventory { + graph: Default::default(), + templates: vec![], + group_candidates: vec![], + }, + configurations: vec![], + }; + let id = append(root, &mut configured); + configured + .inventory + .graph + .query_roots + .push(QueryRootCandidates { + compatible_candidates: vec![id], + }); + let original = vec![QueryRoot::Operator(original)]; + let entries = vec![QueryWorkloadEntry { + query: Query(query.into()), + requirements: QueryRequirements { + accuracy: target, + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + 60_000, + ))), + range_evaluation: None, + }]; + let context = CandidatePlanningContext { + deadline: None, + refine_materialization: false, + original_roots: &original, + entries: &entries, + data: &data, + models: crate::PlanningModels::builtin(), + calibration: crate::CalibrationFit::default(), + }; + let profile = extract_price_options(&configured, &context) + .unwrap() + .unwrap(); + assert!( + !profile.candidates[id.0].is_empty(), + "two certified sketches must reach root" + ); + let proposal = + crate::search::milp::direct_price_options::solve_joint_price_proposal_with_metrics( + &configured.inventory.graph, + &[], + None, + &mut Default::default(), + &profile, + ) + .unwrap() + .unwrap(); + let mut rejections = crate::search::refinement::planner::CandidateRejectionSummary::default(); + let evaluated = crate::search::refinement::planner::evaluate_configured_candidate( + &configured, + &context, + &proposal.selected, + 0, + &mut rejections, + &mut Vec::new(), + ) + .unwrap() + .unwrap_or_else(|| panic!("two-sketch plan rejected: {rejections:?}")); + let actual = evaluated.0.costs[&evaluated.0.selected] + .cost_per_second + .unwrap(); + assert!((actual - proposal.direct_modeled_cost.unwrap()).abs() <= 1e-9 * actual.max(1e-15)); +} diff --git a/crates/plan-selection/src/search/costing/range_price_options_tests.rs b/crates/plan-selection/src/search/costing/range_price_options_tests.rs new file mode 100644 index 000000000..93a3c7f87 --- /dev/null +++ b/crates/plan-selection/src/search/costing/range_price_options_tests.rs @@ -0,0 +1,266 @@ +//! Whole-evaluator parity for selected consumer range-retention tiers. +use super::*; +use crate::search::candidates::configurations::CandidateConfiguration; +use crate::search::candidates::graph::{ + CandidateDependencyGraph, PhysicalCandidate, ProducerRequirement, QueryRootCandidates, +}; +use crate::search::candidates::inventory::CandidateInventory; +use asap_logical_asap_optimization::memo::MemoGroupId; +use asap_logical_asap_optimization::per_computation_candidates::{ + compose_logical_candidate, generate_per_computation_candidates, ComputationCandidate, +}; +use asap_logical_asap_optimization::test_support::lower_promql; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::ir::schema::SketchAlgorithm; +use asap_types::ir::OperatorNode; +use asap_types::workload::{ + DataWorkload, Predictability, Query, QueryRequirements, QueryWorkloadEntry, RangeEvaluation, + RepetitionInterval, +}; +use std::rc::Rc; + +#[test] +fn shared_pane_range_tiers_match_evaluator_for_ingestion_and_kept_refresh() { + fn append( + node: Rc, + configured: &mut CandidateConfigurationInventory, + ingestion: bool, + mode: Materialization, + ) -> PhysicalCandidateId { + let ingestion = ingestion + || (matches!(node.asap(), Some(ASAPOp::PaneChain { .. })) + && mode == Materialization::MatAtIngestion(Medium::Memory)); + let pane = matches!(node.asap(), Some(ASAPOp::PaneChain { .. })); + let mut inputs = Vec::new(); + node.operator + .map_children(|child| inputs.push(append(child.clone(), configured, ingestion, mode))); + let id = PhysicalCandidateId(configured.configurations.len()); + configured.configurations.push(CandidateConfiguration { + source_candidate: id, + materialization: if pane { + mode + } else { + Materialization::NotMaterialized + }, + timing: if ingestion { + ExecutionTiming::IngestionTime + } else { + ExecutionTiming::QueryTime + }, + }); + configured.inventory.templates.push(Some(node)); + configured + .inventory + .graph + .candidates + .push(PhysicalCandidate { + group: MemoGroupId(id.0 as u32), + local_cost: 0.0, + producer_requirements: inputs + .into_iter() + .map(|supplier| ProducerRequirement { + compatible_producers: vec![supplier].into(), + }) + .collect(), + }); + id + } + + for mode in [ + Materialization::MatAtIngestion(Medium::Memory), + Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory), + ] { + let target = AccuracyTarget::max_error(0.05) + .failure_probability(0.01) + .accuracy_scope(AccuracyScope::PerValue); + let original = lower_promql("quantile_over_time(0.95, latency[5m])", target.clone()); + let generated = + generate_per_computation_candidates(vec![(0, QueryRoot::Operator(original.clone()))]) + .unwrap(); + let choice = generated.targets[0] + .choices + .candidates + .iter() + .position(|candidate| { + matches!( + candidate, + ComputationCandidate::Summary(SketchAlgorithm::Kll) + ) + }) + .unwrap(); + let roots: Vec<_> = compose_logical_candidate(&generated, &[choice]) + .unwrap() + .into_iter() + .map(|(_, root)| root) + .collect(); + let rules = asap_logical_asap_optimization::asap_aware_cse::pane_candidates( + &roots, + &[Some(10_000)], + ); + let roots = + asap_logical_asap_optimization::asap_aware_cse::window_composition_pass2_candidates( + &roots, &rules, + ) + .unwrap() + .pop() + .unwrap() + .roots + .unwrap(); + let data: DataWorkload = serde_json::from_value(serde_json::json!({ + "arrival":{"ingesting":{"rows_per_second":100.0}}, + "sources":{"latency":{"series_count":2,"rows_held":30000,"values_non_negative":true}} + })) + .unwrap(); + let sized = + crate::resize_for_failure_probability(&roots, std::slice::from_ref(&target), &data) + .unwrap(); + let mut configured = CandidateConfigurationInventory { + inventory: CandidateInventory { + graph: CandidateDependencyGraph::default(), + templates: vec![], + group_candidates: vec![], + }, + configurations: vec![], + }; + let QueryRoot::Operator(root) = &sized[0] else { + panic!("operator") + }; + let root_id = append(root.clone(), &mut configured, false, mode); + configured.inventory.graph.query_roots = vec![ + QueryRootCandidates { + compatible_candidates: vec![root_id] + }; + 2 + ]; + let originals = vec![ + QueryRoot::Operator(original.clone()), + QueryRoot::Operator(original), + ]; + let entries: Vec<_> = [(60_000, 120_000, 10_000), (90_000, 600_000, 30_000)] + .into_iter() + .map(|(interval, span_ms, step_ms)| QueryWorkloadEntry { + query: Query("quantile_over_time(0.95, latency[5m])".into()), + requirements: QueryRequirements { + accuracy: target.clone(), + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence: QueryRecurrence::Repeated(RepeatedDemand::FixedInterval( + RepetitionInterval( + if mode == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory) + { + 10_000 + } else { + interval + }, + ), + )), + range_evaluation: Some(RangeEvaluation { span_ms, step_ms }), + }) + .collect(); + let mut settings = crate::PlanSelectionSettings::default(); + settings.resource_speed.ms_per_cpu_op *= 1000.0; + settings.resource_speed.ms_per_scanned_byte *= 1000.0; + settings.measured_operation_costs.ns_per_evaluation = 1_000_000.0; + let deployment = asap_logical_asap_optimization::DeploymentCapabilities { + query_time_results_retained: true, + ..Default::default() + }; + let context = CandidatePlanningContext { + deadline: None, + refine_materialization: false, + original_roots: &originals, + entries: &entries, + data: &data, + models: crate::PlanningModels { + plan_selection: &settings, + deployment_capabilities: &deployment, + ..crate::PlanningModels::builtin() + }, + calibration: crate::CalibrationFit::default(), + }; + let profile = extract_price_options(&configured, &context) + .unwrap() + .unwrap(); + let chain = configured + .inventory + .templates + .iter() + .position(|node| { + node.as_ref() + .is_some_and(|node| matches!(node.asap(), Some(ASAPOp::PaneChain { .. }))) + }) + .unwrap(); + let tiers: Vec<_> = profile.candidates[chain] + .iter() + .map(|option| option.required_range_span_ms) + .collect(); + assert!( + tiers.contains(&Some(120_000)) && tiers.contains(&Some(600_000)), + "{mode:?}: {tiers:?}" + ); + let general = profile.schedules.as_ref().unwrap(); + let kept = mode == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory); + assert!( + (general.union_rate([true, true].into_iter()) - if kept { 2.9 } else { 0.45 }).abs() + < 1e-12 + ); + assert!( + (general.request_union_rate([true, true].into_iter()) + - if kept { 0.1 } else { 1.0 / 60.0 + 1.0 / 90.0 }) + .abs() + < 1e-12 + ); + assert!(profile.candidates[chain] + .iter() + .all(|option| option.profile.per_request + == (mode == Materialization::MatAtQueryKeptAcrossEvaluations(Medium::Memory)))); + let proposal = + crate::search::milp::direct_price_options::solve_joint_price_proposal_with_metrics( + &configured.inventory.graph, + &[], + None, + &mut Default::default(), + &profile, + ) + .unwrap() + .unwrap(); + // Keep option indices/bindings unchanged, but deny every option a long + // enough retention witness. Both outputs must then be infeasible. + let mut insufficient = profile.clone(); + for option in &mut insufficient.candidates[chain] { + option.required_range_span_ms = Some(120_000); + } + assert!( + crate::search::milp::direct_price_options::solve_joint_price_proposal_with_metrics( + &configured.inventory.graph, + &[], + None, + &mut Default::default(), + &insufficient + ) + .unwrap() + .is_none() + ); + let mut rejections = + crate::search::refinement::planner::CandidateRejectionSummary::default(); + let evaluated = crate::search::refinement::planner::evaluate_configured_candidate( + &configured, + &context, + &proposal.selected, + 0, + &mut rejections, + &mut Vec::new(), + ) + .unwrap() + .unwrap_or_else(|| panic!("{mode:?}: {rejections:?}")); + let actual = evaluated.0.costs[&evaluated.0.selected] + .cost_per_second + .unwrap(); + let modeled = proposal.direct_modeled_cost.unwrap(); + assert!( + (actual - modeled).abs() <= 1e-9 * actual.max(1e-15), + "{mode:?}: direct {modeled}, evaluator {actual}" + ); + } +} diff --git a/crates/plan-selection/src/search/costing/supplier_price_options.rs b/crates/plan-selection/src/search/costing/supplier_price_options.rs index 322e9f4c6..cf1e30748 100644 --- a/crates/plan-selection/src/search/costing/supplier_price_options.rs +++ b/crates/plan-selection/src/search/costing/supplier_price_options.rs @@ -17,12 +17,20 @@ struct PriceClass<'a> { work: WorkBounds, workers: Option, compaction: f64, - bindings: Vec, + bindings: Vec>, + ancestor_work: std::collections::BTreeMap, + sketch_ids: std::collections::BTreeSet, + sketch_sources: std::collections::BTreeMap>, + range_span_ms: Option, } impl PriceClass<'_> { fn same_interface(&self, other: &Self) -> bool { - same_price(&self.priced, &other.priced) + self.ancestor_work == other.ancestor_work + && self.sketch_ids == other.sketch_ids + && self.sketch_sources == other.sketch_sources + && self.range_span_ms == other.range_span_ms + && same_price(&self.priced, &other.priced) && self.lookback == other.lookback && self.sketches == other.sketches && self.raw_source == other.raw_source @@ -36,6 +44,45 @@ impl PriceClass<'_> { /// Exact scan-lookback equality remains required: max-width sharing needs an /// explicit selected-consumer witness. Cyclic catalogs use bounded local /// interface propagation; selected physical graphs must still be acyclic. +struct InputTuples { + choices: Vec>, + indices: Vec, + done: bool, +} +impl InputTuples { + fn new(choices: Vec>) -> Self { + Self { + indices: vec![0; choices.len()], + done: choices.iter().any(Vec::is_empty), + choices, + } + } +} +impl Iterator for InputTuples { + type Item = Vec; + fn next(&mut self) -> Option { + if self.done { + return None; + } + let tuple = self + .choices + .iter() + .zip(&self.indices) + .map(|(choices, index)| choices[*index]) + .collect(); + self.done = true; + for position in (0..self.indices.len()).rev() { + self.indices[position] += 1; + if self.indices[position] < self.choices[position].len() { + self.done = false; + break; + } + self.indices[position] = 0; + } + Some(tuple) + } +} + pub(in crate::search) fn extract_price_options( configured: &CandidateConfigurationInventory, context: &CandidatePlanningContext<'_>, @@ -104,6 +151,7 @@ pub(in crate::search) fn extract_price_options( let mut compactions = vec![0.0; count]; let mut classes: Vec>> = vec![Vec::new(); count]; let mut total = 0; + let mut total_bindings = 0usize; // Every legal unary DAG path has at most N candidates. N monotone passes // therefore cover every supported simple path even in a cyclic catalog. // Walk-derived bindings are harmless: the solver enforces original-node @@ -115,110 +163,275 @@ pub(in crate::search) fn extract_price_options( if nodes[id].is_none() { continue; } - let mut supplier_counts = Vec::new(); - if let Some(input) = configured.inventory.graph.candidates[id] - .producer_requirements - .first() - { + let mut inputs = Vec::new(); + for input in &configured.inventory.graph.candidates[id].producer_requirements { + let mut choices = Vec::new(); for producer in &input.compatible_producers { check_deadline(context.deadline)?; - if producer.0 != id { - supplier_counts.push((Some(*producer), classes[producer.0].len())); + if producer.0 == id { + continue; + } + for price_option in 0..classes[producer.0].len() { + choices.push(SupplierPriceBinding { + producer: *producer, + price_option, + }); } } - } else { - supplier_counts.push((None, 1)); + inputs.push(choices); } - // Stream local bindings instead of allocating a potentially enormous - // supplier-by-option product before the deadline can be checked. - let inputs = supplier_counts.into_iter().flat_map(|(producer, count)| { - (0..count).map(move |price_option| { - producer.map(|producer| SupplierPriceBinding { - producer, - price_option, - }) - }) - }); - for binding in inputs { - for &lookback in &all_lookbacks[id] { - check_deadline(context.deadline)?; - if let Some(binding) = binding { - let supplier = binding.producer.0; - let source = &classes[supplier][binding.price_option]; - cache[supplier] = Some(source.priced.clone()); - lookbacks[supplier] = source.lookback; - sketch_counts[supplier] = source.sketches; - raw_sources[supplier] = source.raw_source.clone(); - closure_work[supplier] = source.work; - worker_counts[supplier] = source.workers; - compactions[supplier] = source.compaction; - } - lookbacks[id] = lookback; - cache[id] = None; - visited[id] = false; - suppliers[id].clear(); - sketch_counts[id] = 0; - raw_sources[id] = None; - closure_work[id] = WorkBounds::default(); - worker_counts[id] = None; - compactions[id] = 0.0; - price( - id, - configured, - context, - &nodes, - &lookbacks, - &mut cache, - &mut visited, - &mut visiting, - &mut suppliers, - &mut sketch_counts, - &mut raw_sources, - &mut closure_work, - &mut worker_counts, - &mut compactions, - delta, - binding.map(|binding| binding.producer), - )?; - let priced = cache[id].take(); - if let Some(binding) = binding { - cache[binding.producer.0] = None; - } - let Some(priced) = priced else { continue }; - let candidate = PriceClass { - priced, - lookback, - sketches: sketch_counts[id], - raw_source: raw_sources[id].clone(), - // Query-time closure work never determines an ingestion - // worker count. Keeping it would distinguish every traversal - // of an identity cycle despite identical local interfaces. - work: if nodes[id].as_ref().unwrap().output_state.timing - == ExecutionTiming::IngestionTime + let node = nodes[id].as_ref().unwrap(); + let spans: Vec> = if matches!( + node.payload, + asap_types::ir::Operator::ASAP(ASAPOp::PaneChain { .. }) + ) { + let mut spans: Vec<_> = context + .entries + .iter() + .map(|entry| Some(entry.range_evaluation.map_or(0, |range| range.span_ms))) + .collect(); + spans.sort(); + spans.dedup(); + spans + } else { + vec![None] + }; + for binding in InputTuples::new(inputs) { + for range_span_ms in &spans { + for &lookback in &all_lookbacks[id] { + check_deadline(context.deadline)?; + let mut selected_sources = std::collections::BTreeMap::new(); + let mut ancestor_work = std::collections::BTreeMap::new(); + let mut sketch_ids = std::collections::BTreeSet::new(); + let mut sketch_sources = std::collections::BTreeMap::new(); + let mut compatible = true; + for binding in &binding { + if selected_sources + .insert(binding.producer.0, binding.price_option) + .is_some_and(|prior| prior != binding.price_option) + { + compatible = false; + break; + } + let source = &classes[binding.producer.0][binding.price_option]; + for (ancestor, work) in &source.ancestor_work { + if ancestor_work + .insert(*ancestor, *work) + .is_some_and(|prior| prior != *work) + { + compatible = false; + } + } + sketch_ids.extend(source.sketch_ids.iter().copied()); + for (sketch, proof) in &source.sketch_sources { + if sketch_sources + .insert(*sketch, proof.clone()) + .is_some_and(|prior| prior != *proof) + { + compatible = false; + } + } + } + if !compatible || ancestor_work.contains_key(&id) { + continue; + } + let upstream_sketches = sketch_ids.len(); + if matches!( + &node.payload, + asap_types::ir::Operator::ASAP( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + } | ASAPOp::PaneChain { + family: FieldDataType::Sketch(..), + .. + } + ) + ) { + sketch_ids.insert(id); + sketch_sources.insert( + id, + binding.first().and_then(|input| { + classes[input.producer.0][input.price_option] + .raw_source + .clone() + }), + ); + } + if !sketch_ids.is_empty() { + let Some(delta) = delta else { continue }; + let divided = delta / sketch_ids.len() as f64; + if !sketch_ids.iter().all(|sketch| { + let template = + configured.inventory.templates[*sketch].as_ref().unwrap(); + if requires_source_sizing(template) { + let Some(source) = + sketch_sources.get(sketch).and_then(|s| s.as_ref()) + else { + return false; + }; + let Some(bound) = context + .data + .sources + .get(source) + .and_then(|facts| facts.max_distinct_per_evaluation) + else { + return false; + }; + let Some( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(kind, _), + .. + } + | ASAPOp::PaneChain { + family: FieldDataType::Sketch(kind, _), + .. + }, + ) = template.asap() + else { + return false; + }; + let SketchParams::Hll { + precision, + relative_error, + } = kind.params() + else { + return false; + }; + hll_size_certified(*precision, *relative_error, divided, bound) + } else { + stable_size(template, Some(divided)) + } + }) { + continue; + } + } + let mut upstream_work = WorkBounds::default(); + for work in ancestor_work.values() { + upstream_work.minimum = (upstream_work.minimum + f64::from_bits(*work)) + .next_down() + .max(0.0); + upstream_work.maximum = + (upstream_work.maximum + f64::from_bits(*work)).next_up(); + } + for binding in &binding { + let supplier = binding.producer.0; + let source = &classes[supplier][binding.price_option]; + cache[supplier] = Some(source.priced.clone()); + lookbacks[supplier] = source.lookback; + sketch_counts[supplier] = source.sketches; + raw_sources[supplier] = source.raw_source.clone(); + closure_work[supplier] = source.work; + worker_counts[supplier] = source.workers; + compactions[supplier] = source.compaction; + } + lookbacks[id] = lookback; + cache[id] = None; + visited[id] = false; + suppliers[id].clear(); + sketch_counts[id] = 0; + raw_sources[id] = None; + closure_work[id] = WorkBounds::default(); + worker_counts[id] = None; + compactions[id] = 0.0; + let input_context = PriceInputs { + suppliers: binding.iter().map(|binding| binding.producer).collect(), + upstream_work, + upstream_sketches, + range_span_ms: *range_span_ms, + }; + price( + id, + configured, + context, + &nodes, + &lookbacks, + &mut cache, + &mut visited, + &mut visiting, + &mut suppliers, + &mut sketch_counts, + &mut raw_sources, + &mut closure_work, + &mut worker_counts, + &mut compactions, + delta, + Some(&input_context), + )?; + let priced = cache[id].take(); + for binding in &binding { + cache[binding.producer.0] = None; + } + let Some(priced) = priced else { continue }; + if matches!( + &node.payload, + asap_types::ir::Operator::ASAP( + ASAPOp::SummaryAgg { + family: FieldDataType::Sketch(..), + .. + } | ASAPOp::PaneChain { + family: FieldDataType::Sketch(..), + .. + } + ) + ) { + sketch_ids.insert(id); + } + if node.output_state.timing == ExecutionTiming::IngestionTime { + ancestor_work.insert(id, priced.work_ms.to_bits()); + } else { + ancestor_work.clear(); + } + let candidate = PriceClass { + priced, + lookback, + sketches: sketch_counts[id], + raw_source: raw_sources[id].clone(), + // Query-time closure work never determines an ingestion + // worker count. Keeping it would distinguish every traversal + // of an identity cycle despite identical local interfaces. + work: if nodes[id].as_ref().unwrap().output_state.timing + == ExecutionTiming::IngestionTime + { + closure_work[id] + } else { + WorkBounds::default() + }, + workers: worker_counts[id], + compaction: compactions[id], + bindings: vec![binding.clone()], + ancestor_work, + sketch_ids, + sketch_sources, + range_span_ms: *range_span_ms, + }; + if let Some(existing) = classes[id] + .iter_mut() + .find(|known| known.same_interface(&candidate)) { - closure_work[id] + for binding in candidate.bindings { + if !existing.bindings.contains(&binding) { + total_bindings += 1; + if total_bindings > 200_000 { + return Err(DirectProfileError::Unsupported("local pricing domain exceeds 200000 correlated input tuples".into())); + } + existing.bindings.push(binding); + } + } } else { - WorkBounds::default() - }, - workers: worker_counts[id], - compaction: compactions[id], - bindings: binding.into_iter().collect(), - }; - if let Some(existing) = classes[id] - .iter_mut() - .find(|known| known.same_interface(&candidate)) - { - for binding in candidate.bindings { - if !existing.bindings.contains(&binding) { - existing.bindings.push(binding); + total_bindings += candidate.bindings.len(); + if total_bindings > 200_000 { + return Err(DirectProfileError::Unsupported( + "local pricing domain exceeds 200000 correlated input tuples" + .into(), + )); } + total += 1; + if total > MAXIMUM_PRICE_OPTIONS { + return Err(DirectProfileError::Unsupported(format!("local pricing domain exceeds {MAXIMUM_PRICE_OPTIONS} interface options"))); + } + classes[id].push(candidate); } - } else { - total += 1; - if total > MAXIMUM_PRICE_OPTIONS { - return Err(DirectProfileError::Unsupported(format!("local pricing domain exceeds {MAXIMUM_PRICE_OPTIONS} interface options"))); - } - classes[id].push(candidate); } } } @@ -241,7 +454,8 @@ pub(in crate::search) fn extract_price_options( class.compaction, context, )?, - compatible_suppliers: class.bindings.clone(), + compatible_input_tuples: class.bindings.clone(), + required_range_span_ms: class.range_span_ms, }) }) .collect::, DirectProfileError>>() @@ -253,6 +467,11 @@ pub(in crate::search) fn extract_price_options( }; let request_ms = settings.measured_operation_costs.ns_per_evaluation * 1e-6; Ok(Some(JointPriceProfile { + query_range_spans_ms: context + .entries + .iter() + .map(|entry| entry.range_evaluation.map_or(0, |range| range.span_ms)) + .collect(), schedules, query_rate: rate, request_latency_seconds: request_ms / 1000.0, diff --git a/crates/plan-selection/src/search/milp/direct_cost.rs b/crates/plan-selection/src/search/milp/direct_cost.rs index 5e4970fc6..b9a2fd125 100644 --- a/crates/plan-selection/src/search/milp/direct_cost.rs +++ b/crates/plan-selection/src/search/milp/direct_cost.rs @@ -1,16 +1,19 @@ //! Exact objective encoding for certified query-time and ingestion profiles. -//! Fixed evaluation schedules may be equal or use a bounded union-rate table; -//! maintained and retained states carry separate standing resource charges. +//! Equal schedules have a compact fast path; mixed recurrence/range demand +//! uses exact evaluator atoms. Retained state carries separate standing charges. //! Profiles are supplied by physical costing; this module never infers prices //! from a logical template or treats accuracy as a monetary penalty. use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; use std::collections::BTreeMap; -#[path = "direct_fixed_schedule.rs"] -mod fixed_schedules; -#[path = "direct_schedule.rs"] -mod schedules; -pub(crate) use fixed_schedules::{FixedQuerySchedule, FixedScheduleLattice}; +#[path = "general_schedule.rs"] +mod general_schedule; +#[path = "general_schedule_rates.rs"] +mod general_schedule_rates; +#[cfg(test)] +#[path = "schedule_selection_tests.rs"] +mod schedule_selection_tests; +pub(crate) use general_schedule_rates::QueryScheduleRates; use crate::search::candidates::graph::*; @@ -28,6 +31,7 @@ const MINIMUM_NORMALIZED_LATENCY_TERM: f64 = 1e-6; #[derive(Debug, Clone, Default)] pub(crate) struct NodeCostProfile { + pub per_request: bool, pub execution_cost: f64, pub ingestion: bool, pub standing_cost_per_second: f64, @@ -49,15 +53,6 @@ impl NodeCostProfile { } } -/// Exact evaluator demand for each nonempty selected query subset. Only small -/// heterogeneous domains are admitted; this is not a workload-plan product. -#[derive(Debug, Clone)] -pub(crate) struct QueryScheduleRates { - pub per_query_rates: Vec, - pub materialized_subset_rates: Vec, - pub compact_fixed: Option, -} - #[derive(Debug, Clone)] pub(crate) struct DirectCostProfile { pub nodes: Vec, @@ -81,6 +76,7 @@ impl DirectCostProfile { || !valid(self.request_cost) || self.nodes.iter().any(|n| { !valid(n.execution_cost) + || (n.per_request && (!n.materialized || n.ingestion)) || !valid(n.standing_cost_per_second) || (n.ingestion && n.held_memory_cost_per_second != 0.0) || !valid(n.compaction_seconds) @@ -126,47 +122,14 @@ impl DirectCostProfile { )); } if let Some(schedule) = &self.schedules { - let queries = graph.query_roots.len(); - if let Some(lattice) = &schedule.compact_fixed { - lattice.validate(schedule, queries)?; - } else { - if queries > 3 { - return Err(CandidateSelectionError::UnsupportedDirectCost( - "heterogeneous query schedule table supports at most three output queries" - .into(), - )); - } - if !schedule.per_query_rates.iter().sum::().is_finite() - || schedule.per_query_rates.len() != queries - || schedule.materialized_subset_rates.len() != (1usize << queries) - || schedule.materialized_subset_rates[0] != 0.0 - || schedule - .per_query_rates - .iter() - .any(|rate| !rate.is_finite() || *rate <= 0.0) - || schedule - .materialized_subset_rates - .iter() - .skip(1) - .any(|rate| !rate.is_finite() || *rate <= 0.0) - || schedule - .per_query_rates - .iter() - .enumerate() - .any(|(q, rate)| *rate != schedule.materialized_subset_rates[1 << q]) - { - return Err(CandidateSelectionError::InvalidGraph( - "invalid direct query-schedule rates".into(), - )); - } - } + schedule.validate(graph.query_roots.len())?; } if let Some(schedule) = &self.schedules { - let total_rate: f64 = schedule.per_query_rates.iter().sum(); + let total_rate: f64 = schedule.per_query_rates().iter().sum(); if schedule - .per_query_rates + .per_query_rates() .iter() - .any(|rate| rate / total_rate < MINIMUM_QUERY_RATE_WEIGHT) + .any(|rate| *rate > 0.0 && rate / total_rate < MINIMUM_QUERY_RATE_WEIGHT) { return Err(CandidateSelectionError::UnsupportedDirectCost( "query-rate weights are below the supported weighted-lifetime numerical range" @@ -208,10 +171,11 @@ impl DirectCostProfile { .filter(|term| *term > 0.0) .reduce(f64::min); let minimum_rate_weight = self.schedules.as_ref().map_or(1.0, |schedule| { - let total: f64 = schedule.per_query_rates.iter().sum(); + let total: f64 = schedule.per_query_rates().iter().sum(); schedule - .per_query_rates + .per_query_rates() .iter() + .filter(|rate| **rate > 0.0) .map(|rate| rate / total) .fold(1.0, f64::min) }); @@ -268,7 +232,7 @@ impl DirectCostProfile { .collect(); let rates = self.schedules.as_ref().map_or_else( || vec![self.query_rate; graph.query_roots.len()], - |schedule| schedule.per_query_rates.clone(), + |schedule| schedule.per_query_rates().to_vec(), ); let union_rate = |id: usize| { if self.nodes[id].ingestion { @@ -277,15 +241,11 @@ impl DirectCostProfile { let Some(schedule) = &self.schedules else { return self.query_rate; }; - if let Some(lattice) = &schedule.compact_fixed { - return lattice.union_rate(reached.iter().map(|row| row[id])); + if self.nodes[id].per_request { + schedule.request_union_rate(reached.iter().map(|row| row[id])) + } else { + schedule.union_rate(reached.iter().map(|row| row[id])) } - let mask = reached - .iter() - .enumerate() - .filter(|(_, row)| row[id]) - .fold(0usize, |mask, (q, _)| mask | (1 << q)); - schedule.materialized_subset_rates[mask] }; let mut execution_rates = vec![0.0; self.nodes.len()]; let boundaries = selected @@ -335,7 +295,7 @@ impl DirectCostProfile { .sum::() / denominator } else { - 0.0 + readers.iter().copied().fold(0.0, f64::max) } }; total += node.standing_cost_per_second @@ -427,23 +387,8 @@ impl DirectCostProfile { let latency_scale = self.maximum_latency(graph).max(f64::MIN_POSITIVE); let rates = self.schedules.as_ref().map_or_else( || vec![self.query_rate], - |schedule| { - if schedule.compact_fixed.is_some() { - return schedule.per_query_rates.clone(); - } - schedule - .materialized_subset_rates - .iter() - .copied() - .filter(|rate| *rate > 0.0) - .collect() - }, + |schedule| schedule.coefficient_rates(), ); - let denominator_scale = self - .schedules - .as_ref() - .and_then(|s| s.compact_fixed.as_ref()) - .map_or(1.0, FixedScheduleLattice::maximum_denominator_scale); self.nodes.iter().flat_map(move |node| { let mut coefficients = Vec::with_capacity(rates.len() * 2 + 1); let node_rates = if node.ingestion { @@ -456,13 +401,6 @@ impl DirectCostProfile { node.execution_cost * rate, node.held_memory_cost_per_second * rate * latency_scale, ]); - // Compact mean-lifetime selectors multiply by total weight / - // selected weight; include that largest possible coefficient. - if !node.materialized && denominator_scale > 1.0 { - coefficients.push( - node.held_memory_cost_per_second * rate * latency_scale * denominator_scale, - ); - } } coefficients.push(node.standing_cost_per_second); coefficients.push(node.raw_retention.as_ref().map_or(0.0, |(_, cost)| *cost)); @@ -647,7 +585,7 @@ pub(super) fn encode( check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, ) -> Result { if let Some(schedule) = &profile.schedules { - return schedules::encode( + return general_schedule::encode( graph, profile, schedule, @@ -863,6 +801,7 @@ mod tests { request_cost: 7.0, nodes: vec![ NodeCostProfile { + per_request: false, ingestion: false, standing_cost_per_second: 0.0, execution_cost: 2.0, @@ -922,6 +861,7 @@ mod tests { ..Default::default() }, NodeCostProfile { + per_request: false, ingestion: false, standing_cost_per_second: 0.0, execution_cost: 1.0, @@ -970,6 +910,7 @@ mod tests { request_cost: 7.0, nodes: vec![ NodeCostProfile { + per_request: false, ingestion: false, standing_cost_per_second: 0.0, execution_cost: materialized_build_cost, @@ -1162,11 +1103,7 @@ mod tests { fn two_ingestion_boundaries_recompute_their_shared_input() { let (graph, mut profile) = ingestion_profile(true, 1.0); assert!((solve(&graph, &profile) - 29.4).abs() < 1e-6); - profile.schedules = Some(QueryScheduleRates { - compact_fixed: None, - per_query_rates: vec![1.0, 2.0], - materialized_subset_rates: vec![0.0, 1.0, 2.0, 3.0], - }); + profile.schedules = Some(schedule_selection_tests::rate_schedule(&[1.0, 2.0])); let proposal = solve_direct_cost_proposal_with_metrics( &graph, &[], @@ -1327,11 +1264,7 @@ mod tests { // refresh; distinct fixed intervals add under the evaluator's policy. for (rates, union, expected) in [(vec![2.0, 2.0], 2.0, 74.4), (vec![1.0, 2.0], 3.0, 83.0)] { if rates[0] != rates[1] { - profile.schedules = Some(QueryScheduleRates { - compact_fixed: None, - per_query_rates: rates.clone(), - materialized_subset_rates: vec![0.0, rates[0], rates[1], union], - }); + profile.schedules = Some(schedule_selection_tests::rate_schedule(&rates)); } let weighted_latency = rates[0] * 3.6 + rates[1] * 4.6; let rate_sum: f64 = rates.iter().sum(); @@ -1458,11 +1391,7 @@ mod tests { }; for rates in [[1.0, 1.0], [1.0, 2.0]] { if rates[0] != rates[1] { - profile.schedules = Some(QueryScheduleRates { - compact_fixed: None, - per_query_rates: rates.to_vec(), - materialized_subset_rates: vec![0.0, rates[0], rates[1], rates[0] + rates[1]], - }); + profile.schedules = Some(schedule_selection_tests::rate_schedule(&rates)); } let expected = (0..4) .map(|mask| { diff --git a/crates/plan-selection/src/search/milp/direct_fixed_schedule.rs b/crates/plan-selection/src/search/milp/direct_fixed_schedule.rs deleted file mode 100644 index 4174e57d0..000000000 --- a/crates/plan-selection/src/search/milp/direct_fixed_schedule.rs +++ /dev/null @@ -1,567 +0,0 @@ -//! Compact fixed-interval accounting follows the evaluator's policy: distinct -//! intervals add, distinct known phases add, unknown phase joins a known phase. -//! Weighted mean lifetimes use reachable integer rate totals, not reader subsets. -use super::*; - -const MAXIMUM_RATE_TICKS: u64 = 4096; - -#[derive(Debug, Clone, Copy)] -pub(crate) struct FixedQuerySchedule { - pub interval_ms: u32, - pub phase_ms: Option, -} - -#[derive(Debug, Clone)] -pub(crate) struct FixedScheduleLattice { - queries: Vec, - weights: Vec, -} - -fn gcd(mut left: u64, mut right: u64) -> u64 { - while right != 0 { - (left, right) = (right, left % right); - } - left -} - -impl QueryScheduleRates { - pub(crate) fn from_fixed_schedules( - queries: Vec, - ) -> Result { - if queries.is_empty() || queries.iter().any(|q| q.interval_ms == 0) { - return Err(CandidateSelectionError::InvalidGraph( - "empty or zero-interval fixed schedule".into(), - )); - } - let mut common = 1u64; - for query in &queries { - let interval = u64::from(query.interval_ms); - common = (common / gcd(common, interval)) - .checked_mul(interval) - .ok_or_else(|| { - CandidateSelectionError::UnsupportedDirectCost( - "fixed schedule lattice arithmetic overflow".into(), - ) - })?; - } - let mut weights: Vec<_> = queries - .iter() - .map(|q| common / u64::from(q.interval_ms)) - .collect(); - let divisor = weights.iter().copied().reduce(gcd).unwrap(); - for weight in &mut weights { - *weight /= divisor; - } - let total = weights - .iter() - .try_fold(0u64, |sum, weight| sum.checked_add(*weight)); - if total.is_none_or(|total| total > MAXIMUM_RATE_TICKS) { - return Err(CandidateSelectionError::UnsupportedDirectCost(format!( - "fixed schedule rate lattice exceeds {MAXIMUM_RATE_TICKS} total ticks" - ))); - } - Ok(Self { - per_query_rates: queries - .iter() - .map(|q| 1000.0 / f64::from(q.interval_ms)) - .collect(), - materialized_subset_rates: Vec::new(), - compact_fixed: Some(FixedScheduleLattice { - queries, - weights: weights.into_iter().map(|w| w as usize).collect(), - }), - }) - } -} - -impl FixedScheduleLattice { - pub(super) fn maximum_denominator_scale(&self) -> f64 { - self.weights.iter().sum::() as f64 - } - - pub(super) fn validate( - &self, - schedule: &QueryScheduleRates, - queries: usize, - ) -> Result<(), CandidateSelectionError> { - let expected = QueryScheduleRates::from_fixed_schedules(self.queries.clone())?; - if queries != self.queries.len() - || expected.per_query_rates != schedule.per_query_rates - || !schedule.materialized_subset_rates.is_empty() - || expected.compact_fixed.as_ref().unwrap().weights != self.weights - { - return Err(CandidateSelectionError::InvalidGraph( - "inconsistent compact fixed schedule".into(), - )); - } - Ok(()) - } - - pub(super) fn union_rate(&self, selected: impl Iterator) -> f64 { - let mut intervals = BTreeMap::)>::new(); - for (query, selected) in self.queries.iter().zip(selected) { - if !selected { - continue; - } - let (unknown, phases) = intervals.entry(query.interval_ms).or_default(); - if let Some(phase) = query.phase_ms { - phases.insert(phase % u64::from(query.interval_ms)); - } else { - *unknown = true; - } - } - intervals - .into_iter() - .map(|(interval, (unknown, phases))| { - phases.len().max(usize::from(unknown)) as f64 * 1000.0 / f64::from(interval) - }) - .sum() - } -} - -fn logical_or( - variables: &mut ProblemVariables, - constraints: &mut Vec, - inputs: &[Variable], -) -> Option { - if inputs.is_empty() { - return None; - } - if inputs.len() == 1 { - return Some(inputs[0]); - } - let value = variables.add(variable().min(0).max(1)); - for input in inputs { - constraints.push(constraint!(value >= *input)); - } - constraints.push(constraint!( - value <= inputs.iter().copied().sum::() - )); - Some(value) -} - -fn logical_and( - variables: &mut ProblemVariables, - constraints: &mut Vec, - left: Variable, - right: Variable, -) -> Variable { - let value = variables.add(variable().min(0).max(1)); - constraints.push(constraint!(value <= left)); - constraints.push(constraint!(value <= right)); - constraints.push(constraint!(value >= left + right - 1)); - value -} - -fn union_terms( - lattice: &FixedScheduleLattice, - node: usize, - reach: &[Vec>], - variables: &mut ProblemVariables, - constraints: &mut Vec, -) -> Vec<(Variable, f64)> { - let mut intervals = BTreeMap::, Vec>>::new(); - for (q, schedule) in lattice.queries.iter().enumerate() { - if let Some(reached) = reach[q][node] { - intervals - .entry(schedule.interval_ms) - .or_default() - .entry( - schedule - .phase_ms - .map(|p| p % u64::from(schedule.interval_ms)), - ) - .or_default() - .push(reached); - } - } - let mut terms = Vec::new(); - for (interval, mut phases) in intervals { - let rate = 1000.0 / f64::from(interval); - let unknown = logical_or( - variables, - constraints, - &phases.remove(&None).unwrap_or_default(), - ); - let known: Vec<_> = phases - .values() - .filter_map(|readers| logical_or(variables, constraints, readers)) - .collect(); - terms.extend(known.iter().map(|phase| (*phase, rate))); - if let Some(unknown) = unknown { - if let Some(any_known) = logical_or(variables, constraints, &known) { - let extra = variables.add(variable().min(0).max(1)); - constraints.push(constraint!(extra <= unknown)); - constraints.push(constraint!(extra <= 1.0 - any_known)); - constraints.push(constraint!(extra >= unknown - any_known)); - terms.push((extra, rate)); - } else { - terms.push((unknown, rate)); - } - } - } - terms -} - -#[allow(clippy::too_many_arguments)] -pub(super) fn encode( - graph: &CandidateDependencyGraph, - profile: &DirectCostProfile, - schedule: &QueryScheduleRates, - lattice: &FixedScheduleLattice, - variables: &mut ProblemVariables, - constraints: &mut Vec, - active: &[Variable], - edges: &[(ProducerConsumerConnection, Variable)], - roots: &[(QueryRootAssignment, Variable)], - reach: &[Vec>], - divisor: f64, - check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, -) -> Result { - let count = graph.candidates.len(); - let queries = graph.query_roots.len(); - let scale = profile.maximum_latency(graph).max(f64::MIN_POSITIVE); - let latency = query_latency_expressions( - graph, - profile, - variables, - constraints, - reach, - &check_deadline, - )?; - let mut unions = Vec::with_capacity(count); - for id in 0..count { - check_deadline()?; - unions.push(union_terms(lattice, id, reach, variables, constraints)); - } - let mut demands = vec![Vec::<(Variable, f64)>::new(); count]; - for flow in boundary_flows( - graph, - profile, - variables, - constraints, - active, - edges, - roots, - &check_deadline, - )? { - for (id, used) in flow.reached.into_iter().enumerate() { - check_deadline()?; - let Some(used) = used else { continue }; - if flow.boundary < queries { - demands[id].push((used, schedule.per_query_rates[flow.boundary])); - } else if profile.nodes[flow.boundary - queries].ingestion { - demands[id].push((used, 1.0)); - } else { - for (phase, rate) in &unions[flow.boundary - queries] { - demands[id].push((logical_and(variables, constraints, used, *phase), *rate)); - } - } - } - } - let mut objective = Expression::from(profile.request_cost / divisor); - let mut raw = BTreeMap::<&str, Vec<(usize, f64)>>::new(); - for (id, node) in profile.nodes.iter().enumerate() { - check_deadline()?; - let terms = if node.materialized { - if node.ingestion { - vec![(active[id], 1.0)] - } else { - unions[id].clone() - } - } else { - demands[id].clone() - }; - objective += node.standing_cost_per_second / divisor * active[id]; - objective += - node.execution_cost / divisor * terms.iter().map(|(v, r)| *r * *v).sum::(); - if let Some((source, cost)) = &node.raw_retention { - raw.entry(source).or_default().push((id, *cost)); - } - if node.held_memory_cost_per_second == 0.0 { - continue; - } - let mut lifetime = Expression::from(0.0); - if node.materialized { - let longest = variables.add(variable().min(0).max(1)); - for (q, row) in reach.iter().enumerate() { - if let Some(reached) = row[id] { - constraints.push(constraint!(longest >= latency[q].clone() - (1.0 - reached))); - } - } - for (used, rate) in terms { - lifetime += rate * gated(variables, constraints, longest.into(), used, 1.0); - } - } else { - let possible: Vec<_> = reach - .iter() - .enumerate() - .filter_map(|(q, row)| row[id].map(|r| (q, r))) - .collect(); - let maximum: usize = possible.iter().map(|(q, _)| lattice.weights[*q]).sum(); - if maximum == 0 { - continue; - } - let mut totals = vec![false; maximum + 1]; - totals[0] = true; - let mut accumulated = 0; - for (q, _) in &possible { - let weight = lattice.weights[*q]; - for sum in (0..=accumulated).rev() { - if totals[sum] { - totals[sum + weight] = true; - } - } - accumulated += weight; - } - // Denominator totals need binary selectors: unlike full reader - // subsets, their weighted-sum equality alone permits mixtures. - let selectors: Vec<_> = totals - .iter() - .enumerate() - .filter(|(_, yes)| **yes) - .map(|(total, _)| (total, variables.add(variable().binary()))) - .collect(); - constraints.push(constraint!( - selectors.iter().map(|(_, v)| *v).sum::() == 1 - )); - constraints.push(constraint!( - selectors - .iter() - .map(|(d, v)| *d as f64 * *v) - .sum::() - == possible - .iter() - .map(|(q, r)| lattice.weights[*q] as f64 * *r) - .sum::() - )); - let numerator: Expression = possible - .iter() - .map(|(q, r)| { - lattice.weights[*q] as f64 / maximum as f64 - * gated(variables, constraints, latency[*q].clone(), *r, 1.0) - }) - .sum(); - for (denominator, selector) in selectors.into_iter().filter(|(d, _)| *d > 0) { - check_deadline()?; - let chosen = gated(variables, constraints, numerator.clone(), selector, 1.0); - for (used, rate) in &terms { - lifetime += *rate * maximum as f64 / denominator as f64 - * gated(variables, constraints, chosen.into(), *used, 1.0); - } - } - } - objective += node.held_memory_cost_per_second * scale / divisor * lifetime; - } - for entries in raw.into_values() { - let held = variables.add(variable().min(0)); - for (id, cost) in entries { - constraints.push(constraint!(held >= cost / divisor * active[id])); - } - objective += held; - } - Ok(objective) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::search::milp::model::{ - solve_direct_cost_proposal_with_metrics, CandidateSolveMetrics, - }; - use asap_logical_asap_optimization::memo::MemoGroupId; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - fn node(input: Option) -> PhysicalCandidate { - PhysicalCandidate { - group: MemoGroupId(0), - local_cost: 0.0, - producer_requirements: input - .into_iter() - .map(|id| ProducerRequirement { - compatible_producers: vec![PhysicalCandidateId(id)].into(), - }) - .collect(), - } - } - - // All reader subsets agree with the existing interval accounting policy; - // the compact representation never constructs a 2^Q rate table. - #[test] - fn compact_fixed_rates_match_evaluator_subsets() { - let intervals = [1000, 2000, 1000, 4000, 2000, 1000]; - let fixed: Vec<_> = intervals - .iter() - .map(|ms| FixedQuerySchedule { - interval_ms: *ms, - phase_ms: None, - }) - .collect(); - let schedule = QueryScheduleRates::from_fixed_schedules(fixed).unwrap(); - let recurrences: Vec<_> = intervals - .iter() - .map(|ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(*ms))) - }) - .collect(); - for mask in 0..1usize << intervals.len() { - let expected = crate::cost::execution_demand::evaluation_rate( - recurrences - .iter() - .enumerate() - .filter(|(q, _)| mask & (1 << q) != 0) - .map(|(_, r)| (r, None)), - 3600.0, - ) - .unwrap(); - let actual = schedule - .compact_fixed - .as_ref() - .unwrap() - .union_rate((0..intervals.len()).map(|q| mask & (1 << q) != 0)); - assert_eq!(actual, expected); - } - assert!(schedule.materialized_subset_rates.is_empty()); - let large = QueryScheduleRates::from_fixed_schedules(vec![ - FixedQuerySchedule { - interval_ms: 1000, - phase_ms: None - }; - 128 - ]) - .unwrap(); - assert_eq!(large.compact_fixed.unwrap().weights.len(), 128); - assert!(matches!( - QueryScheduleRates::from_fixed_schedules(vec![ - FixedQuerySchedule { - interval_ms: 1, - phase_ms: None - }, - FixedQuerySchedule { - interval_ms: 100000, - phase_ms: None - }, - ]), - Err(CandidateSelectionError::UnsupportedDirectCost(_)) - )); - } - - // Known phases add; unknown phase adds only when no known phase is active. - #[test] - fn compact_phase_activation_is_exact_for_every_subset() { - let schedule = QueryScheduleRates::from_fixed_schedules(vec![ - FixedQuerySchedule { - interval_ms: 1000, - phase_ms: None, - }, - FixedQuerySchedule { - interval_ms: 1000, - phase_ms: Some(0), - }, - FixedQuerySchedule { - interval_ms: 1000, - phase_ms: Some(500), - }, - FixedQuerySchedule { - interval_ms: 1000, - phase_ms: Some(1500), - }, - ]) - .unwrap(); - for mask in 0..16 { - let mut variables = ProblemVariables::new(); - let mut constraints = Vec::new(); - let reach: Vec<_> = (0..4) - .map(|q| { - let value = variables.add(variable().binary()); - constraints.push(constraint!(value == f64::from(mask & (1 << q) != 0))); - vec![Some(value)] - }) - .collect(); - let lattice = schedule.compact_fixed.as_ref().unwrap(); - let terms = union_terms(lattice, 0, &reach, &mut variables, &mut constraints); - let expression: Expression = terms.iter().map(|(v, r)| *r * *v).sum(); - use good_lp::{highs, Solution, SolverModel}; - let solution = variables - .minimise(-expression.clone()) - .using(highs) - .with_all(constraints) - .solve() - .unwrap(); - let expected = lattice.union_rate((0..4).map(|q| mask & (1 << q) != 0)); - assert_eq!(solution.eval(&expression), expected); - } - } - - // Five readers share a query-materialized build. Upstream lifetime uses - // their weighted mean; the build uses their maximum and union refresh rate. - #[test] - fn compact_schedule_prices_shared_refresh_and_weighted_memory() { - let intervals = [1000, 2000, 1000, 4000, 2000]; - let schedule = QueryScheduleRates::from_fixed_schedules( - intervals - .iter() - .map(|ms| FixedQuerySchedule { - interval_ms: *ms, - phase_ms: None, - }) - .collect(), - ) - .unwrap(); - let mut candidates = vec![node(None), node(Some(0))]; - candidates.extend((0..5).map(|_| node(Some(1)))); - let graph = CandidateDependencyGraph { - candidates, - query_roots: (2..7) - .map(|id| QueryRootCandidates { - compatible_candidates: vec![PhysicalCandidateId(id)], - }) - .collect(), - ..Default::default() - }; - let mut nodes = vec![ - NodeCostProfile { - execution_cost: 2.0, - work_seconds: 1.0, - held_memory_cost_per_second: 3.0, - ..Default::default() - }, - NodeCostProfile { - execution_cost: 4.0, - work_seconds: 2.0, - held_memory_cost_per_second: 2.0, - materialized: true, - ..Default::default() - }, - ]; - nodes.extend((0..5).map(|q| NodeCostProfile { - execution_cost: 1.0, - work_seconds: q as f64 + 1.0, - ..Default::default() - })); - let rates = schedule.per_query_rates.clone(); - let union = 1.75; - let weighted: f64 = rates - .iter() - .enumerate() - .map(|(q, r)| r * (q as f64 + 4.0)) - .sum(); - let sum: f64 = rates.iter().sum(); - let expected = union * (6.0 + 3.0 * weighted / sum + 2.0 * 8.0) + sum; - let profile = DirectCostProfile { - nodes, - schedules: Some(schedule), - query_rate: 1.0, - request_latency_seconds: 0.0, - request_cost: 0.0, - }; - let mut metrics = CandidateSolveMetrics::default(); - let proposal = - solve_direct_cost_proposal_with_metrics(&graph, &[], None, &mut metrics, &profile) - .unwrap() - .unwrap(); - assert!((proposal.direct_modeled_cost.unwrap() - expected).abs() < 1e-8); - assert!((proposal.model_lower_bound.unwrap() - expected).abs() < 1e-6); - assert!(metrics.peak_variables < 5000); - } -} diff --git a/crates/plan-selection/src/search/milp/direct_price_options.rs b/crates/plan-selection/src/search/milp/direct_price_options.rs index e734d49d2..8d2a90752 100644 --- a/crates/plan-selection/src/search/milp/direct_price_options.rs +++ b/crates/plan-selection/src/search/milp/direct_price_options.rs @@ -12,7 +12,8 @@ pub(crate) struct SupplierPriceBinding { #[derive(Debug, Clone)] pub(crate) struct CandidatePriceOption { pub profile: NodeCostProfile, - pub compatible_suppliers: Vec, + pub compatible_input_tuples: Vec>, + pub required_range_span_ms: Option, } #[derive(Debug, Clone)] @@ -22,12 +23,13 @@ pub(crate) struct JointPriceProfile { pub request_latency_seconds: f64, pub request_cost: f64, pub candidates: Vec>, + pub query_range_spans_ms: Vec, } use super::direct_cost::DirectCostProfile; use super::model::{CandidateProposal, CandidateSolveMetrics, ObservedAssignment}; use crate::search::candidates::graph::*; -use good_lp::{constraint, Constraint, Expression, Variable}; +use good_lp::{constraint, variable, Constraint, Expression, ProblemVariables, Variable}; /// Internal option nodes share an original physical identity. At most one may /// activate, so two readers cannot obtain incompatible views of one producer. @@ -36,6 +38,9 @@ pub(super) struct PriceOptionProjection<'a> { origins: Vec, options: Vec>, observations: &'a [ObservedAssignment], + input_tuples: Vec<(PhysicalCandidateId, Vec>)>, + range_contexts: Vec<(PhysicalCandidateId, u64)>, + query_spans: &'a [u64], } impl PriceOptionProjection<'_> { @@ -81,6 +86,77 @@ impl PriceOptionProjection<'_> { } } + pub(super) fn constrain_input_tuples( + &self, + variables: &mut ProblemVariables, + active: &[Variable], + edges: &[(ProducerConsumerConnection, Variable)], + constraints: &mut Vec, + check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, + ) -> Result<(), CandidateSelectionError> { + let mut by_consumer = std::collections::BTreeMap::<_, Vec<_>>::new(); + for (edge, selected) in edges { + check_deadline()?; + by_consumer + .entry(edge.consumer) + .or_default() + .push((edge, selected)); + } + for (consumer, tuples) in &self.input_tuples { + check_deadline()?; + let mut selectors = Vec::new(); + let mut support = std::collections::BTreeMap::<_, Expression>::new(); + for tuple in tuples { + check_deadline()?; + let selector = variables.add(variable().binary()); + selectors.push(selector); + for (position, producer) in tuple.iter().enumerate() { + *support + .entry((position, *producer)) + .or_insert_with(|| Expression::from(0.0)) += selector; + } + } + constraints.push(constraint!( + selectors.iter().copied().sum::() == active[consumer.0] + )); + for (edge, selected) in by_consumer.get(consumer).into_iter().flatten() { + check_deadline()?; + let matches = support + .get(&(edge.requirement, edge.producer)) + .cloned() + .unwrap_or_else(|| Expression::from(0.0)); + constraints.push(constraint!(**selected == matches)); + } + } + Ok(()) + } + + pub(super) fn constrain_reader_contexts( + &self, + active: &[Variable], + reach: &[Vec>], + constraints: &mut Vec, + check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, + ) -> Result<(), CandidateSelectionError> { + for (candidate, span) in &self.range_contexts { + check_deadline()?; + let mut witnesses = Expression::from(0.0); + for (query, row) in reach.iter().enumerate() { + check_deadline()?; + if let Some(reached) = row[candidate.0] { + if self.query_spans[query] > *span { + constraints.push(constraint!(reached == 0)); + } + if self.query_spans[query] == *span { + witnesses += reached; + } + } + } + constraints.push(constraint!(active[candidate.0] <= witnesses)); + } + Ok(()) + } + /// A rejected physical DAG remains rejected across all of its price-option /// encodings. Cuts refer to sums over the original activation/edge identity. pub(super) fn constrain_observations( @@ -163,6 +239,9 @@ pub(crate) fn solve_joint_price_proposal_with_metrics( origins: Vec::new(), options: Vec::new(), observations, + input_tuples: Vec::new(), + range_contexts: Vec::new(), + query_spans: &profile.query_range_spans_ms, }; let mut expanded = CandidateDependencyGraph::default(); let mut nodes = Vec::new(); @@ -186,38 +265,65 @@ pub(crate) fn solve_joint_price_proposal_with_metrics( )); } let required = &graph.candidates[id].producer_requirements; - if !options.is_empty() && required.len() > 1 { - return Err(invalid( - "joint unary price profile has multiple required inputs", - )); - } for (index, option) in options.iter().enumerate() { - if required.is_empty() && !option.compatible_suppliers.is_empty() { - return Err(invalid("source price option has supplier bindings")); + let consumer = projection.options[id][index]; + if let Some(span) = option.required_range_span_ms { + if profile.query_range_spans_ms.len() != graph.query_roots.len() { + return Err(invalid("range contexts do not match query roots")); + } + projection.range_contexts.push((consumer, span)); } - if let Some(input) = required.first() { - let mut suppliers = Vec::new(); - for binding in &option.compatible_suppliers { - if !input.compatible_producers.contains(&binding.producer) { - return Err(invalid("price option references incompatible supplier")); + let mut tuples = Vec::new(); + for tuple in &option.compatible_input_tuples { + if tuple.len() != required.len() { + return Err(invalid("price input tuple arity mismatch")); + } + let mut mapped = Vec::new(); + for (position, binding) in tuple.iter().enumerate() { + if !required[position] + .compatible_producers + .contains(&binding.producer) + { + return Err(invalid("price tuple references incompatible supplier")); } - let supplier = projection - .options - .get(binding.producer.0) - .and_then(|options| options.get(binding.price_option)) - .ok_or_else(|| { - invalid("price option references missing supplier option") - })?; - if !suppliers.contains(supplier) { - suppliers.push(*supplier); + mapped.push( + *projection + .options + .get(binding.producer.0) + .and_then(|options| options.get(binding.price_option)) + .ok_or_else(|| { + invalid("price tuple references missing supplier option") + })?, + ); + } + if !tuples.contains(&mapped) { + tuples.push(mapped); + } + } + for position in 0..required.len() { + let mut suppliers = Vec::new(); + for tuple in &tuples { + if !suppliers.contains(&tuple[position]) { + suppliers.push(tuple[position]); } } - expanded.candidates[projection.options[id][index].0] + expanded.candidates[consumer.0] .producer_requirements .push(ProducerRequirement { compatible_producers: suppliers.into(), }); } + // Unary choices and single tuples are already exact under the + // existing one-producer-per-input constraints; no extra binaries. + let cartesian_size = expanded.candidates[consumer.0] + .producer_requirements + .iter() + .try_fold(1usize, |size, input| { + size.checked_mul(input.compatible_producers.len()) + }); + if required.len() > 1 && tuples.len() > 1 && cartesian_size != Some(tuples.len()) { + projection.input_tuples.push((consumer, tuples)); + } } } for root in &graph.query_roots { @@ -390,11 +496,14 @@ mod tests { materialized, ..Default::default() }, - compatible_suppliers: producer + required_range_span_ms: None, + compatible_input_tuples: producer .into_iter() - .map(|(id, price_option)| SupplierPriceBinding { - producer: PhysicalCandidateId(id), - price_option, + .map(|(id, price_option)| { + vec![SupplierPriceBinding { + producer: PhysicalCandidateId(id), + price_option, + }] }) .collect(), } @@ -419,6 +528,7 @@ mod tests { ..Default::default() }; let profile = JointPriceProfile { + query_range_spans_ms: vec![], schedules: None, query_rate: 1.0, request_latency_seconds: 0.0, @@ -536,4 +646,54 @@ mod tests { .unwrap(); assert_eq!(proposal.direct_modeled_cost, Some(106.0)); } + // Tuple correlation forbids mixing the cheap first input from one tuple + // with the cheap second input of another tuple. + #[test] + fn correlated_inputs_cannot_form_an_unpriced_cartesian_combination() { + let mut consumer = candidate(&[0, 1]); + consumer + .producer_requirements + .push(consumer.producer_requirements[0].clone()); + let graph = CandidateDependencyGraph { + candidates: vec![candidate(&[]), candidate(&[]), consumer], + query_roots: vec![QueryRootCandidates { + compatible_candidates: vec![PhysicalCandidateId(2)], + }], + ..Default::default() + }; + let binding = |id| SupplierPriceBinding { + producer: PhysicalCandidateId(id), + price_option: 0, + }; + let mut reader = option(1.0, false, None); + reader.compatible_input_tuples = + vec![vec![binding(0), binding(1)], vec![binding(1), binding(0)]]; + let profile = JointPriceProfile { + candidates: vec![ + vec![option(0.0, true, None)], + vec![option(10.0, true, None)], + vec![reader], + ], + schedules: None, + query_rate: 1.0, + request_latency_seconds: 0.0, + request_cost: 0.0, + query_range_spans_ms: vec![0], + }; + let solved = solve_joint_price_proposal_with_metrics( + &graph, + &[], + None, + &mut Default::default(), + &profile, + ) + .unwrap() + .unwrap(); + assert!((solved.direct_modeled_cost.unwrap() - 11.0).abs() < 1e-9); + assert_eq!(solved.selected.connections.len(), 2); + assert_ne!( + solved.selected.connections[0].producer, + solved.selected.connections[1].producer + ); + } } diff --git a/crates/plan-selection/src/search/milp/direct_schedule.rs b/crates/plan-selection/src/search/milp/direct_schedule.rs deleted file mode 100644 index 2c55dc2d1..000000000 --- a/crates/plan-selection/src/search/milp/direct_schedule.rs +++ /dev/null @@ -1,528 +0,0 @@ -//! Heterogeneous execution demand for a small query-subset domain. -//! Subset rates come from the evaluator, not a second scheduling policy. -use super::*; - -#[allow(clippy::too_many_arguments)] -pub(super) fn encode( - graph: &CandidateDependencyGraph, - profile: &DirectCostProfile, - schedule: &QueryScheduleRates, - variables: &mut ProblemVariables, - constraints: &mut Vec, - active: &[Variable], - edges: &[(ProducerConsumerConnection, Variable)], - roots: &[(QueryRootAssignment, Variable)], - reach: &[Vec>], - divisor: f64, - check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, -) -> Result { - if let Some(lattice) = &schedule.compact_fixed { - return fixed_schedules::encode( - graph, - profile, - schedule, - lattice, - variables, - constraints, - active, - edges, - roots, - reach, - divisor, - check_deadline, - ); - } - let count = graph.candidates.len(); - let queries = graph.query_roots.len(); - let subsets = 1usize << queries; - let latency_scale = profile.maximum_latency(graph).max(f64::MIN_POSITIVE); - let latency = query_latency_expressions( - graph, - profile, - variables, - constraints, - reach, - &check_deadline, - )?; - let mut masks = Vec::with_capacity(count); - for id in 0..count { - check_deadline()?; - let allowed = reach - .iter() - .enumerate() - .filter(|(_, row)| row[id].is_some()) - .fold(0usize, |mask, (q, _)| mask | (1 << q)); - let row: Vec<_> = (0..subsets) - .map(|mask| (mask & !allowed == 0).then(|| variables.add(variable().min(0).max(1)))) - .collect(); - // Integral query reach fixes exactly one subset, so these selector - // auxiliaries need no additional integer branching. - constraints.push(constraint!( - row.iter().flatten().copied().sum::() == 1 - )); - constraints.push(constraint!( - row.iter().skip(1).flatten().copied().sum::() == active[id] - )); - for (query, reachable) in reach.iter().enumerate() { - let included: Expression = row - .iter() - .enumerate() - .filter(|(mask, _)| mask & (1 << query) != 0) - .filter_map(|(_, variable)| *variable) - .sum(); - constraints.push(constraint!( - included == reachable[id].map_or_else(|| Expression::from(0.0), Expression::from) - )); - } - masks.push(row); - } - // Each pair is (binary demand indicator, executions per second). A query - // boundary has its own rate; a materialized boundary uses its selected - // downstream query subset's union rate and stops upstream propagation. - let mut demands = vec![Vec::<(Variable, f64)>::new(); count]; - for flow in boundary_flows( - graph, - profile, - variables, - constraints, - active, - edges, - roots, - &check_deadline, - )? { - let boundary = flow.boundary; - for (id, reaches) in flow.reached.into_iter().enumerate() { - check_deadline()?; - let Some(reaches) = reaches else { continue }; - if boundary < queries { - demands[id].push((reaches, schedule.per_query_rates[boundary])); - } else if profile.nodes[boundary - queries].ingestion { - demands[id].push((reaches, 1.0)); - } else { - for (mask, selector) in masks[boundary - queries].iter().enumerate().skip(1) { - let Some(selector) = selector else { continue }; - let used = variables.add(variable().min(0).max(1)); - constraints.push(constraint!(used <= reaches)); - constraints.push(constraint!(used <= *selector)); - constraints.push(constraint!(used >= reaches + *selector - 1)); - demands[id].push((used, schedule.materialized_subset_rates[mask])); - } - } - } - } - let mut objective = Expression::from(profile.request_cost / divisor); - let mut retained = BTreeMap::<&str, Vec<(usize, f64)>>::new(); - for (id, node) in profile.nodes.iter().enumerate() { - check_deadline()?; - let executions: Expression = if node.materialized && node.ingestion { - active[id].into() - } else if node.materialized { - masks[id] - .iter() - .enumerate() - .skip(1) - .filter_map(|(mask, selector)| { - selector.map(|s| schedule.materialized_subset_rates[mask] * s) - }) - .sum() - } else { - demands[id].iter().map(|(used, rate)| *rate * *used).sum() - }; - objective += node.standing_cost_per_second / divisor * active[id]; - objective += node.execution_cost / divisor * executions; - if let Some((source, cost)) = &node.raw_retention { - retained.entry(source).or_default().push((id, *cost)); - } - if node.held_memory_cost_per_second == 0.0 { - continue; - } - let mut lifetime_demand = Expression::from(0.0); - if node.materialized { - let longest = variables.add(variable().min(0).max(1)); - for (query, row) in reach.iter().enumerate() { - if let Some(reached) = row[id] { - constraints.push(constraint!( - longest >= latency[query].clone() - (1.0 - reached) - )); - } - } - for (mask, selector) in masks[id].iter().enumerate().skip(1) { - if let Some(selector) = selector { - lifetime_demand += schedule.materialized_subset_rates[mask] - * gated(variables, constraints, longest.into(), *selector, 1.0); - } - } - } else { - for (mask, selector) in masks[id].iter().enumerate().skip(1) { - check_deadline()?; - let Some(selector) = selector else { continue }; - let total_rate: f64 = schedule - .per_query_rates - .iter() - .enumerate() - .filter(|(q, _)| mask & (1 << q) != 0) - .map(|(_, rate)| *rate) - .sum(); - let mean: Expression = latency - .iter() - .enumerate() - .filter(|(q, _)| mask & (1 << q) != 0) - .map(|(q, latency)| schedule.per_query_rates[q] / total_rate * latency.clone()) - .sum(); - let selected_mean = gated(variables, constraints, mean, *selector, 1.0); - for (demand, rate) in &demands[id] { - check_deadline()?; - lifetime_demand += - *rate * gated(variables, constraints, selected_mean.into(), *demand, 1.0); - } - } - } - objective += node.held_memory_cost_per_second * latency_scale / divisor * lifetime_demand; - } - for entries in retained.into_values() { - let retained_cost = variables.add(variable().min(0)); - for (id, cost) in entries { - constraints.push(constraint!(retained_cost >= cost / divisor * active[id])); - } - objective += retained_cost; - } - Ok(objective) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::search::milp::model::{ - solve_direct_cost_proposal_with_metrics, CandidateSolveMetrics, - }; - use asap_logical_asap_optimization::memo::MemoGroupId; - use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; - - fn schedules(queries: &[QueryRecurrence]) -> QueryScheduleRates { - let union = |mask: usize| { - crate::cost::execution_demand::evaluation_rate( - queries - .iter() - .enumerate() - .filter(|(q, _)| mask & (1 << q) != 0) - .map(|(_, query)| (query, None)), - 3600.0, - ) - .unwrap() - }; - QueryScheduleRates { - compact_fixed: None, - per_query_rates: (0..queries.len()).map(|q| union(1 << q)).collect(), - materialized_subset_rates: (0..1 << queries.len()).map(union).collect(), - } - } - fn node(input: Option) -> PhysicalCandidate { - PhysicalCandidate { - group: MemoGroupId(0), - local_cost: 0.0, - producer_requirements: input - .into_iter() - .map(|id| ProducerRequirement { - compatible_producers: vec![PhysicalCandidateId(id)].into(), - }) - .collect(), - } - } - fn root(id: usize) -> QueryRootCandidates { - QueryRootCandidates { - compatible_candidates: vec![PhysicalCandidateId(id)], - } - } - - // Boundary demand is the evaluator's union rate; upstream held memory uses - // the rate-weighted mean, not the arithmetic mean or the sum of root rates. - #[test] - fn heterogeneous_intervals_price_boundary_rates_and_weighted_lifetimes() { - let queries = [500, 1000].map(|ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(ms))) - }); - let graph = CandidateDependencyGraph { - candidates: vec![node(None), node(Some(0)), node(Some(1)), node(Some(1))], - query_roots: vec![root(2), root(3)], - ..Default::default() - }; - let profile = DirectCostProfile { - schedules: Some(schedules(&queries)), - query_rate: 1.0, - request_latency_seconds: 0.5, - request_cost: 7.0, - nodes: vec![ - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 2.0, - work_seconds: 1.0, - compaction_seconds: 0.0, - held_memory_cost_per_second: 3.0, - ..Default::default() - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 1.0, - work_seconds: 2.0, - compaction_seconds: 0.0, - held_memory_cost_per_second: 4.0, - materialized: true, - raw_retention: None, - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - work_seconds: 3.0, - compaction_seconds: 0.0, - ..Default::default() - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - work_seconds: 4.0, - compaction_seconds: 0.0, - ..Default::default() - }, - ], - }; - assert_eq!( - profile - .schedules - .as_ref() - .unwrap() - .materialized_subset_rates[3], - 3.0 - ); - let result = solve_direct_cost_proposal_with_metrics( - &graph, - &[], - None, - &mut CandidateSolveMetrics::default(), - &profile, - ) - .unwrap() - .unwrap(); - assert!((result.direct_modeled_cost.unwrap() - 167.5).abs() < 1e-8); - assert!((result.model_lower_bound.unwrap() - 167.5).abs() < 1e-8); - } - - // Same-interval readers can share one evaluation only at compatible phases. - // Use the actual evaluator's table rather than assuming all equal rates merge. - #[test] - fn aligned_and_offset_phases_use_evaluator_union_policy() { - use asap_types::workload::TimestampMs; - let graph = CandidateDependencyGraph { - candidates: vec![node(None)], - query_roots: vec![root(0), root(0)], - ..Default::default() - }; - for (second_phase, expected) in [(0, 7.0), (500, 14.0)] { - let queries = [0, second_phase].map(|phase| { - QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { - interval: RepetitionInterval(1000), - evaluation_phase: TimestampMs(phase), - }) - }); - let profile = DirectCostProfile { - schedules: Some(schedules(&queries)), - query_rate: 1.0, - request_latency_seconds: 0.0, - request_cost: 0.0, - nodes: vec![NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 3.0, - work_seconds: 2.0, - compaction_seconds: 0.0, - held_memory_cost_per_second: 2.0, - materialized: true, - raw_retention: None, - }], - }; - let result = solve_direct_cost_proposal_with_metrics( - &graph, - &[], - None, - &mut CandidateSolveMetrics::default(), - &profile, - ) - .unwrap() - .unwrap(); - assert!((result.direct_modeled_cost.unwrap() - expected).abs() < 1e-8); - // HiGHS epigraph feasibility is tolerance-based; the decoded selected - // cost above is recomputed independently and remains exact here. - assert!( - (result.model_lower_bound.unwrap() - expected).abs() <= 1e-7 * expected, - "bound={:?}, expected={expected}", - result.model_lower_bound - ); - } - } - - // Every reader can independently choose either materialized producer. - // Compare the joint optimum with four independently priced assignments. - #[test] - fn heterogeneous_selection_matches_exhaustive_reader_assignments() { - let queries = [500, 1000].map(|ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(ms))) - }); - let schedule = schedules(&queries); - let mut graph = CandidateDependencyGraph { - candidates: vec![node(None), node(None), node(Some(0)), node(Some(0))], - query_roots: vec![root(2), root(3)], - ..Default::default() - }; - for id in [2, 3] { - graph.candidates[id].producer_requirements[0].compatible_producers = - vec![PhysicalCandidateId(0), PhysicalCandidateId(1)].into(); - } - let profile = DirectCostProfile { - schedules: Some(schedule.clone()), - query_rate: 1.0, - request_latency_seconds: 0.5, - request_cost: 7.0, - nodes: vec![ - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 10.0, - work_seconds: 1.0, - compaction_seconds: 0.0, - held_memory_cost_per_second: 3.0, - materialized: true, - raw_retention: None, - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 2.0, - work_seconds: 0.5, - compaction_seconds: 0.0, - held_memory_cost_per_second: 1.0, - materialized: true, - raw_retention: None, - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 1.0, - work_seconds: 2.0, - compaction_seconds: 0.0, - ..Default::default() - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 1.0, - work_seconds: 3.0, - compaction_seconds: 0.0, - ..Default::default() - }, - ], - }; - let expected = [[0, 0], [0, 1], [1, 0], [1, 1]] - .into_iter() - .map(|assignment| { - let latency = [ - 2.5 + [1.0, 0.5][assignment[0]], - 3.5 + [1.0, 0.5][assignment[1]], - ]; - let mut cost = 7.0 + 3.0; - for supplier in [0, 1] { - let mask = assignment - .iter() - .enumerate() - .filter(|(_, chosen)| **chosen == supplier) - .fold(0, |mask, (q, _)| mask | (1 << q)); - let longest = latency - .iter() - .enumerate() - .filter(|(q, _)| mask & (1 << q) != 0) - .map(|(_, time)| *time) - .fold(0.0, f64::max); - cost += schedule.materialized_subset_rates[mask] - * ([10.0, 2.0][supplier] + [3.0, 1.0][supplier] * longest); - } - cost - }) - .fold(f64::INFINITY, f64::min); - let result = solve_direct_cost_proposal_with_metrics( - &graph, - &[], - None, - &mut CandidateSolveMetrics::default(), - &profile, - ) - .unwrap() - .unwrap(); - assert!((result.direct_modeled_cost.unwrap() - expected).abs() < 1e-8); - // HiGHS epigraph feasibility is tolerance-based; the decoded selected - // cost above is recomputed independently and remains exact here. - assert!( - (result.model_lower_bound.unwrap() - expected).abs() <= 1e-7 * expected, - "bound={:?}, expected={expected}", - result.model_lower_bound - ); - } - // A rare slow reader's weighted held lifetime can dominate cost despite a - // tiny rate. Reject that numerically unsupported mean before optimization, - // rather than erase its contribution in a relaxed equality. - #[test] - fn extreme_reader_rates_are_explicitly_unsupported() { - let queries = [1, u32::MAX].map(|ms| { - QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(ms))) - }); - let graph = CandidateDependencyGraph { - candidates: vec![node(None), node(None), node(Some(0)), node(Some(0))], - query_roots: vec![root(2), root(3)], - ..Default::default() - }; - let profile = DirectCostProfile { - schedules: Some(schedules(&queries)), - query_rate: 1.0, - request_latency_seconds: 0.0, - request_cost: 0.0, - nodes: vec![ - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 1.0, - held_memory_cost_per_second: 1e6, - ..Default::default() - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - execution_cost: 2.0, - held_memory_cost_per_second: 1e6, - ..Default::default() - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - work_seconds: 1.0, - compaction_seconds: 0.0, - ..Default::default() - }, - NodeCostProfile { - ingestion: false, - standing_cost_per_second: 0.0, - work_seconds: 1e9, - compaction_seconds: 0.0, - ..Default::default() - }, - ], - }; - let rates = &profile.schedules.as_ref().unwrap().per_query_rates; - assert!( - rates[1] * 1e9 * 1e6 > 1e8, - "the rare reader has meaningful memory cost" - ); - assert!(matches!( - profile.validate(&graph), - Err(CandidateSelectionError::UnsupportedDirectCost(_)) - )); - } -} diff --git a/crates/plan-selection/src/search/milp/general_schedule.rs b/crates/plan-selection/src/search/milp/general_schedule.rs new file mode 100644 index 000000000..dd29c5c1e --- /dev/null +++ b/crates/plan-selection/src/search/milp/general_schedule.rs @@ -0,0 +1,178 @@ +//! Exact mixed-schedule demand and weighted lifetimes without subset tables. +use super::*; + +fn both( + variables: &mut ProblemVariables, + constraints: &mut Vec, + left: Variable, + right: Variable, +) -> Variable { + let value = variables.add(variable().min(0).max(1)); + constraints.push(constraint!(value <= left)); + constraints.push(constraint!(value <= right)); + constraints.push(constraint!(value >= left + right - 1)); + value +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn encode( + graph: &CandidateDependencyGraph, + profile: &DirectCostProfile, + schedule: &QueryScheduleRates, + variables: &mut ProblemVariables, + constraints: &mut Vec, + active: &[Variable], + edges: &[(ProducerConsumerConnection, Variable)], + roots: &[(QueryRootAssignment, Variable)], + reach: &[Vec>], + divisor: f64, + check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, +) -> Result { + let count = graph.candidates.len(); + let queries = graph.query_roots.len(); + let latency_scale = profile.maximum_latency(graph).max(f64::MIN_POSITIVE); + let latency = query_latency_expressions( + graph, + profile, + variables, + constraints, + reach, + &check_deadline, + )?; + let mut materialized = vec![Vec::new(); count]; + for (id, node) in profile.nodes.iter().enumerate() { + check_deadline()?; + if node.materialized { + materialized[id] = if node.ingestion { + vec![(active[id], 1.0)] + } else { + schedule.encode_terms( + id, + reach, + variables, + constraints, + &check_deadline, + node.per_request, + )? + }; + } + } + let mut demands = materialized.clone(); + for flow in boundary_flows( + graph, + profile, + variables, + constraints, + active, + edges, + roots, + &check_deadline, + )? { + for (id, reached) in flow.reached.into_iter().enumerate() { + check_deadline()?; + let Some(reached) = reached else { continue }; + if flow.boundary < queries { + if schedule.per_query_rates()[flow.boundary] > 0.0 { + demands[id].push((reached, schedule.per_query_rates()[flow.boundary])); + } + } else { + for (used, rate) in &materialized[flow.boundary - queries] { + demands[id].push((both(variables, constraints, reached, *used), *rate)); + } + } + } + } + let mut objective = Expression::from(profile.request_cost / divisor); + let mut retained = BTreeMap::<&str, Vec<(usize, f64)>>::new(); + for (id, node) in profile.nodes.iter().enumerate() { + check_deadline()?; + objective += node.standing_cost_per_second / divisor * active[id]; + for (used, rate) in &demands[id] { + objective += node.execution_cost * *rate / divisor * *used; + } + if let Some((source, cost)) = &node.raw_retention { + retained.entry(source).or_default().push((id, *cost)); + } + if node.held_memory_cost_per_second == 0.0 || demands[id].is_empty() { + continue; + } + let lifetime = variables.add(variable().min(0).max(1)); + if node.materialized { + for (q, row) in reach.iter().enumerate() { + if let Some(reached) = row[id] { + constraints.push(constraint!( + lifetime >= latency[q].clone() - (1.0 - reached) + )); + } + } + } else { + // D*A=N is linear after gating A and each query latency by binary + // reach: sum(w_q * reached_q * A) = sum(w_q * reached_q * L_q). + // No reciprocal, LCM, or enumeration of selected reader subsets. + let total: f64 = schedule.per_query_rates().iter().sum(); + let mut left = Expression::from(0.0); + let mut right = Expression::from(0.0); + if total > 0.0 { + for (q, row) in reach.iter().enumerate() { + check_deadline()?; + if let Some(reached) = row[id] { + let weight = schedule.per_query_rates()[q] / total; + if weight == 0.0 { + continue; + } + left += + weight * gated(variables, constraints, lifetime.into(), reached, 1.0); + right += weight + * gated(variables, constraints, latency[q].clone(), reached, 1.0); + } + } + } + constraints.push(constraint!(left == right)); + if schedule.per_query_rates().contains(&0.0) { + let positive: Vec<_> = reach + .iter() + .enumerate() + .filter(|(q, _)| schedule.per_query_rates()[*q] > 0.0) + .filter_map(|(_, row)| row[id]) + .collect(); + let has_positive = variables.add(variable().min(0).max(1)); + for reached in &positive { + constraints.push(constraint!(has_positive >= *reached)); + } + constraints.push(constraint!( + has_positive <= positive.iter().copied().sum::() + )); + // Match the evaluator's longest-reader fallback when every reached + // reader has zero expected demand; no division by zero is needed. + for (q, row) in reach.iter().enumerate() { + if let Some(reached) = row[id] { + constraints.push(constraint!( + lifetime >= latency[q].clone() - (1.0 - reached) - has_positive + )); + } + } + } + } + for (used, rate) in &demands[id] { + objective += node.held_memory_cost_per_second * latency_scale * *rate / divisor + * gated(variables, constraints, lifetime.into(), *used, 1.0); + } + } + for costs in retained.values() { + check_deadline()?; + let maximum = costs + .iter() + .map(|(_, cost)| *cost / divisor) + .fold(0.0, f64::max); + let cost = variables.add(variable().min(0).max(maximum)); + for (id, charge) in costs { + constraints.push(constraint!(cost >= *charge / divisor * active[*id])); + } + objective += cost; + } + Ok(objective) +} + +#[cfg(test)] +#[path = "general_schedule_tests.rs"] +mod tests; diff --git a/crates/plan-selection/src/search/milp/general_schedule_rates.rs b/crates/plan-selection/src/search/milp/general_schedule_rates.rs new file mode 100644 index 000000000..f76ad6bce --- /dev/null +++ b/crates/plan-selection/src/search/milp/general_schedule_rates.rs @@ -0,0 +1,771 @@ +//! Exact finite demand atoms for the evaluator's recurrence accounting policy. +//! Different fixed intervals add; an unknown phase joins the first selected +//! known phase, estimated rates add, and nonrepeating demand takes a maximum. +use crate::cost::execution_demand::evaluation_rate; +use crate::search::candidates::graph::CandidateSelectionError; +use asap_types::workload::{QueryRecurrence, QueryWorkloadEntry, RangeEvaluation, RepeatedDemand}; +use good_lp::{constraint, variable, Constraint, ProblemVariables, Variable}; +use std::collections::BTreeMap; +use std::time::Instant; + +const MAXIMUM_ENUMERATED_OFFSETS: usize = 100_000; +const MAXIMUM_OFFSET_VISITS: usize = 500_000; + +#[derive(Debug, Clone)] +struct Atom { + readers: Vec, + count: u64, +} +#[derive(Debug, Clone)] +struct FixedPhase { + readers: Vec, + atoms: Vec, +} +#[derive(Debug, Clone)] +struct FixedInterval { + rate: f64, + unknown: Vec, + unknown_atoms: Vec, + // Ascending phase order is semantically significant. + known: Vec, +} +#[derive(Debug, Clone, Default)] +struct DemandAtoms { + fixed: Vec, + estimated: Vec<(usize, f64)>, + batch: Vec<(u64, Vec)>, + horizon: f64, +} +#[derive(Debug, Clone)] +pub(crate) struct QueryScheduleRates { + queries: Vec<(QueryRecurrence, Option)>, + rates: Vec, + evaluations: DemandAtoms, + requests: DemandAtoms, +} +fn unsupported(reason: &str) -> CandidateSelectionError { + CandidateSelectionError::UnsupportedDirectCost(reason.into()) +} +fn check(deadline: Option) -> Result<(), CandidateSelectionError> { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + Err(CandidateSelectionError::Solver( + "time budget exhausted constructing schedule atoms".into(), + )) + } else { + Ok(()) + } +} + +// With a common step, k-ranges are nested and can be counted without expanding +// even a billion individual timestamps. Mixed steps retain exact coverage sets. +fn coverage( + readers: &[(usize, u64, u64)], + deadline: Option, +) -> Result, CandidateSelectionError> { + if readers.is_empty() { + return Ok(Vec::new()); + } + if readers.iter().all(|(_, step, _)| *step == readers[0].1) { + let mut endpoints: Vec<_> = readers + .iter() + .map(|(_, step, span)| span / step + 1) + .collect(); + endpoints.sort_unstable(); + endpoints.dedup(); + let mut previous = 0; + let mut atoms = Vec::new(); + for end in endpoints { + check(deadline)?; + let active = readers + .iter() + .filter(|(_, step, span)| span / step + 1 >= end) + .map(|(id, _, _)| *id) + .collect(); + atoms.push(Atom { + readers: active, + count: end - previous, + }); + previous = end; + } + return Ok(atoms); + } + // A few distinct arithmetic progressions can be counted exactly without + // enumerating their offsets. Duplicate geometries share all their readers. + let mut geometries = BTreeMap::<(u64, u64), Vec>::new(); + for &(id, step, span) in readers { + check(deadline)?; + geometries.entry((step, span)).or_default().push(id); + } + if geometries.len() <= 12 { + return arithmetic_coverage(&geometries.into_iter().collect::>(), deadline); + } + let mut offsets: BTreeMap> = BTreeMap::new(); + let mut visits = 0usize; + for &(id, step, span) in readers { + let count = span / step + 1; + if count > MAXIMUM_OFFSET_VISITS as u64 + || visits.saturating_add(count as usize) > MAXIMUM_OFFSET_VISITS + { + return Err(unsupported( + "mixed-step range demand exceeds the exact offset construction budget", + )); + } + visits += count as usize; + for k in 0..count { + check(deadline)?; + offsets.entry(k * step).or_default().push(id); + if offsets.len() > MAXIMUM_ENUMERATED_OFFSETS { + return Err(unsupported( + "mixed-step range demand exceeds the exact offset domain", + )); + } + } + } + let mut classes = BTreeMap::, u64>::new(); + for (_, mut readers) in offsets { + check(deadline)?; + readers.sort_unstable(); + readers.dedup(); + *classes.entry(readers).or_default() += 1; + } + Ok(classes + .into_iter() + .map(|(readers, count)| Atom { readers, count }) + .collect()) +} + +fn arithmetic_coverage( + geometries: &[((u64, u64), Vec)], + deadline: Option, +) -> Result, CandidateSelectionError> { + let size = 1usize << geometries.len(); + let mut counts = vec![0i128; size]; + for (mask, count) in counts.iter_mut().enumerate().skip(1) { + check(deadline)?; + let intersection: Vec<_> = geometries + .iter() + .enumerate() + .filter(|(bit, _)| mask & (1 << bit) != 0) + .map(|(_, (geometry, _))| *geometry) + .collect(); + *count = crate::cost::execution_demand::offset_intersection_count(&intersection); + } + // Superset Möbius inversion changes intersection counts into counts of + // offsets covered by exactly each geometry subset. + for bit in 0..geometries.len() { + for mask in 1..size { + check(deadline)?; + if mask & (1 << bit) == 0 { + counts[mask] -= counts[mask | (1 << bit)]; + } + } + } + let mut atoms = Vec::new(); + for (mask, count) in counts.into_iter().enumerate().skip(1) { + check(deadline)?; + if count == 0 { + continue; + } + let count = u64::try_from(count) + .map_err(|_| unsupported("arithmetic range coverage count overflow"))?; + let mut readers: Vec<_> = geometries + .iter() + .enumerate() + .filter(|(bit, _)| mask & (1 << bit) != 0) + .flat_map(|(_, (_, readers))| readers.iter().copied()) + .collect(); + readers.sort_unstable(); + readers.dedup(); + atoms.push(Atom { readers, count }); + } + Ok(atoms) +} + +impl DemandAtoms { + fn new( + queries: &[(QueryRecurrence, Option)], + horizon: f64, + per_request: bool, + deadline: Option, + ) -> Result { + let mut fixed = BTreeMap::, Vec<(usize, u64, u64)>>>::new(); + let mut estimated = Vec::new(); + let mut batch = BTreeMap::>::new(); + for (id, (recurrence, range)) in queries.iter().enumerate() { + check(deadline)?; + let range = if per_request { None } else { *range }; + let steps = range.map_or(1, |range| range.evaluations_per_request()); + if steps > (1u64 << 53) { + return Err(unsupported( + "range evaluation count exceeds exact floating-point integer precision", + )); + } + match recurrence { + QueryRecurrence::Repeated( + RepeatedDemand::FixedInterval(interval) + | RepeatedDemand::FixedIntervalAt { interval, .. }, + ) => { + let phase = match recurrence { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + evaluation_phase, + .. + }) => Some(evaluation_phase.0 % u64::from(interval.0)), + _ => None, + }; + let (step, span) = range + .filter(|_| steps > 1) + .map_or((u64::from(interval.0), 0), |r| (r.step_ms, r.span_ms)); + fixed + .entry(interval.0) + .or_default() + .entry(phase) + .or_default() + .push((id, step, span)); + } + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(estimate)) => { + estimated.push((id, estimate.expected_rate.0 * steps as f64)) + } + _ => { + let requests = match recurrence { + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(times)) => { + times.len() as u64 + } + QueryRecurrence::OneTime { invocations, .. } => *invocations, + _ => 1, + }; + let count = requests + .checked_mul(steps) + .ok_or_else(|| unsupported("batch invocation count overflow"))?; + if count > (1u64 << 53) { + return Err(unsupported( + "batch invocation count exceeds exact floating-point integer precision", + )); + } + if count > 0 { + batch.entry(count).or_default().push(id); + } + } + } + } + let mut intervals = Vec::new(); + for (interval, mut phases) in fixed { + let unknown = phases.remove(&None).unwrap_or_default(); + let unknown_atoms = coverage(&unknown, deadline)?; + let mut known = Vec::new(); + for (_, readers) in phases { + let members = readers.iter().map(|(id, _, _)| *id).collect(); + let mut together = readers; + together.extend_from_slice(&unknown); + known.push(FixedPhase { + readers: members, + atoms: coverage(&together, deadline)?, + }); + } + intervals.push(FixedInterval { + rate: 1000.0 / f64::from(interval), + unknown: unknown.iter().map(|(id, _, _)| *id).collect(), + unknown_atoms, + known, + }); + } + Ok(Self { + fixed: intervals, + estimated, + batch: batch.into_iter().collect(), + horizon, + }) + } +} + +impl QueryScheduleRates { + pub(crate) fn new( + entries: &[QueryWorkloadEntry], + horizon: f64, + deadline: Option, + ) -> Result { + check(deadline)?; + if entries.is_empty() || !horizon.is_finite() || horizon <= 0.0 { + return Err(CandidateSelectionError::InvalidGraph( + "invalid general schedule horizon or empty workload".into(), + )); + } + let queries: Vec<_> = entries + .iter() + .map(|entry| (entry.recurrence.clone(), entry.range_evaluation)) + .collect(); + for (recurrence, range) in &queries { + check(deadline)?; + if range.is_some_and(|r| r.step_ms == 0 || r.span_ms == u64::MAX) { + return Err(unsupported("invalid or overflowing range schedule")); + } + if matches!(recurrence,QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(interval) | RepeatedDemand::FixedIntervalAt{interval,..}) if interval.0==0) + { + return Err(CandidateSelectionError::InvalidGraph( + "zero fixed interval".into(), + )); + } + } + // Build checked integer products before calling the evaluator oracle. + let evaluations = DemandAtoms::new(&queries, horizon, false, deadline)?; + let requests = DemandAtoms::new(&queries, horizon, true, deadline)?; + let rates = queries + .iter() + .map(|(r, range)| { + evaluation_rate([(r, *range)], horizon) + .map_err(|e| CandidateSelectionError::InvalidGraph(e.to_string())) + }) + .collect::, _>>()?; + if rates.iter().any(|rate| !rate.is_finite() || *rate < 0.0) { + return Err(unsupported("nonfinite general schedule rate")); + } + Ok(Self { + queries, + rates, + evaluations, + requests, + }) + } + pub(crate) fn validate(&self, queries: usize) -> Result<(), CandidateSelectionError> { + if queries != self.queries.len() { + return Err(CandidateSelectionError::InvalidGraph( + "inconsistent general schedule rates".into(), + )); + } + Ok(()) + } + pub(crate) fn coefficient_rates(&self) -> Vec { + let mut result = Vec::new(); + for atoms in [&self.evaluations, &self.requests] { + for interval in &atoms.fixed { + result.extend( + interval + .known + .iter() + .flat_map(|phase| &phase.atoms) + .chain(&interval.unknown_atoms) + .map(|atom| interval.rate * atom.count as f64), + ); + } + result.extend(atoms.estimated.iter().map(|(_, rate)| *rate)); + let mut previous = 0; + for (count, _) in &atoms.batch { + result.push((*count - previous) as f64 / atoms.horizon); + previous = *count; + } + } + result.retain(|rate| *rate > 0.0); + result + } + pub(crate) fn per_query_rates(&self) -> &[f64] { + &self.rates + } + pub(crate) fn union_rate(&self, selected: impl Iterator) -> f64 { + self.rate(selected, false) + } + pub(crate) fn request_union_rate(&self, selected: impl Iterator) -> f64 { + self.rate(selected, true) + } + fn rate(&self, selected: impl Iterator, per_request: bool) -> f64 { + let selected: Vec<_> = selected.collect(); + let atoms = if per_request { + &self.requests + } else { + &self.evaluations + }; + let any = |readers: &[usize]| { + readers + .iter() + .any(|q| selected.get(*q).copied().unwrap_or(false)) + }; + let mut total = 0.0; + for interval in &atoms.fixed { + let first = interval.known.iter().position(|phase| any(&phase.readers)); + for (index, phase) in interval.known.iter().enumerate() { + for atom in &phase.atoms { + if atom.readers.iter().any(|q| { + selected.get(*q).copied().unwrap_or(false) + && (!interval.unknown.contains(q) || first == Some(index)) + }) { + total += interval.rate * atom.count as f64; + } + } + } + if first.is_none() { + for atom in &interval.unknown_atoms { + if any(&atom.readers) { + total += interval.rate * atom.count as f64; + } + } + } + } + total += atoms + .estimated + .iter() + .filter(|(q, _)| selected.get(*q).copied().unwrap_or(false)) + .map(|(_, rate)| *rate) + .sum::(); + total += atoms + .batch + .iter() + .filter(|(_, readers)| any(readers)) + .map(|(count, _)| *count) + .max() + .unwrap_or(0) as f64 + / atoms.horizon; + total + } + pub(crate) fn encode_terms( + &self, + node: usize, + reach: &[Vec>], + variables: &mut ProblemVariables, + constraints: &mut Vec, + check_deadline: impl Fn() -> Result<(), CandidateSelectionError>, + per_request: bool, + ) -> Result, CandidateSelectionError> { + let atoms = if per_request { + &self.requests + } else { + &self.evaluations + }; + let mut terms = Vec::new(); + let readers = |ids: &[usize]| { + ids.iter() + .filter_map(|q| reach[*q][node]) + .collect::>() + }; + for interval in &atoms.fixed { + check_deadline()?; + let mut prior = None; + for phase in &interval.known { + let present = or(variables, constraints, &readers(&phase.readers)); + let first = present.map(|present| without(variables, constraints, present, prior)); + for atom in &phase.atoms { + check_deadline()?; + let (unknown, known): (Vec<&usize>, Vec<&usize>) = atom + .readers + .iter() + .partition(|q| interval.unknown.contains(q)); + let known: Vec<_> = known.into_iter().filter_map(|q| reach[*q][node]).collect(); + let unknown: Vec<_> = unknown + .into_iter() + .filter_map(|q| reach[*q][node]) + .collect(); + let mut occupied = or(variables, constraints, &known) + .into_iter() + .collect::>(); + if let (Some(first), Some(unknown)) = + (first, or(variables, constraints, &unknown)) + { + occupied.push(and(variables, constraints, first, unknown)); + } + if let Some(value) = or(variables, constraints, &occupied) { + terms.push((value, interval.rate * atom.count as f64)); + } + } + prior = or( + variables, + constraints, + &prior.into_iter().chain(present).collect::>(), + ); + } + for atom in &interval.unknown_atoms { + check_deadline()?; + if let Some(unknown) = or(variables, constraints, &readers(&atom.readers)) { + terms.push(( + without(variables, constraints, unknown, prior), + interval.rate * atom.count as f64, + )); + } + } + } + for &(q, rate) in &atoms.estimated { + check_deadline()?; + if rate > 0.0 { + if let Some(reach) = reach[q][node] { + terms.push((reach, rate)); + } + } + } + let mut previous = 0; + for (count, _) in &atoms.batch { + check_deadline()?; + let members: Vec<_> = atoms + .batch + .iter() + .filter(|(other, _)| other >= count) + .flat_map(|(_, ids)| readers(ids)) + .collect(); + if let Some(value) = or(variables, constraints, &members) { + terms.push((value, (*count - previous) as f64 / atoms.horizon)); + } + previous = *count; + } + Ok(terms) + } +} +fn or( + variables: &mut ProblemVariables, + constraints: &mut Vec, + inputs: &[Variable], +) -> Option { + if inputs.is_empty() { + return None; + } + if inputs.len() == 1 { + return Some(inputs[0]); + } + let value = variables.add(variable().min(0).max(1)); + for input in inputs { + constraints.push(constraint!(value >= *input)); + } + constraints.push(constraint!( + value <= inputs.iter().copied().sum::() + )); + Some(value) +} +fn and( + variables: &mut ProblemVariables, + constraints: &mut Vec, + left: Variable, + right: Variable, +) -> Variable { + let value = variables.add(variable().min(0).max(1)); + constraints.push(constraint!(value <= left)); + constraints.push(constraint!(value <= right)); + constraints.push(constraint!(value >= left + right - 1)); + value +} +fn without( + variables: &mut ProblemVariables, + constraints: &mut Vec, + value: Variable, + excluded: Option, +) -> Variable { + let Some(excluded) = excluded else { + return value; + }; + let result = variables.add(variable().min(0).max(1)); + constraints.push(constraint!(result <= value)); + constraints.push(constraint!(result <= 1.0 - excluded)); + constraints.push(constraint!(result >= value - excluded)); + result +} + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::ir::properties::AccuracyTarget; + use asap_types::workload::{ + DemandEstimate, Predictability, Query, QueryRequirements, Rate, RepetitionInterval, + TimestampMs, + }; + use good_lp::{highs, Expression, Solution, SolverModel}; + + fn entry(recurrence: QueryRecurrence, range: Option<(u64, u64)>) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("sum(m)".into()), + requirements: QueryRequirements { + accuracy: AccuracyTarget::exact(), + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence, + range_evaluation: range.map(|(span_ms, step_ms)| RangeEvaluation { span_ms, step_ms }), + } + } + fn fixed(interval: u32, phase: Option, range: Option<(u64, u64)>) -> QueryWorkloadEntry { + entry( + QueryRecurrence::Repeated(match phase { + None => RepeatedDemand::FixedInterval(RepetitionInterval(interval)), + Some(phase) => RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(interval), + evaluation_phase: TimestampMs(phase), + }, + }), + range, + ) + } + fn compare_all_subsets(entries: &[QueryWorkloadEntry]) { + let schedule = QueryScheduleRates::new(entries, 60.0, None).unwrap(); + for per_request in [false, true] { + for mask in 0..(1usize << entries.len()) { + let mut variables = ProblemVariables::new(); + let mut constraints = Vec::new(); + let reach: Vec<_> = (0..entries.len()) + .map(|q| { + let v = variables.add(variable().binary()); + constraints.push(constraint!(v == f64::from(mask & (1 << q) != 0))); + vec![Some(v)] + }) + .collect(); + let terms = schedule + .encode_terms( + 0, + &reach, + &mut variables, + &mut constraints, + || Ok(()), + per_request, + ) + .unwrap(); + let expression: Expression = terms.iter().map(|(v, rate)| *rate * *v).sum(); + // Maximise to prove that an auxiliary rate cannot be inflated; + // all OR/first-phase/max relations are exact equalities in use. + let solution = variables + .minimise(-expression.clone()) + .using(highs) + .with_all(constraints) + .solve() + .unwrap(); + let expected = evaluation_rate( + entries + .iter() + .enumerate() + .filter(|(q, _)| mask & (1 << q) != 0) + .map(|(_, entry)| { + ( + &entry.recurrence, + if per_request { + None + } else { + entry.range_evaluation + }, + ) + }), + 60.0, + ) + .unwrap(); + let actual = solution.eval(&expression); + let decoded = schedule.rate( + (0..entries.len()).map(|q| mask & (1 << q) != 0), + per_request, + ); + assert!( + (decoded - expected).abs() <= 1e-10 * expected.max(1.0), + "decoded {decoded} != oracle {expected}" + ); + assert!( + (actual - expected).abs() <= 1e-10 * expected.max(1.0), + "mask={mask}, requests={per_request}: {actual} != {expected}" + ); + } + } + } + #[test] + fn fixed_ranges_join_unknown_to_first_selected_known_phase() { + compare_all_subsets(&[ + fixed(1000, None, Some((6, 2))), + fixed(1000, Some(0), Some((6, 3))), + fixed(1000, Some(500), Some((6, 2))), + fixed(2000, Some(0), Some((6, 2))), + ]); + } + #[test] + fn mixed_estimates_scheduled_batches_and_unknown_match_evaluator() { + let estimate = |rate| { + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(rate), + observed_at: None, + valid_for: None, + })) + }; + compare_all_subsets(&[ + fixed(997, None, Some((10, 3))), + entry(estimate(0.137), Some((8, 2))), + entry(estimate(0.193), None), + entry( + QueryRecurrence::Repeated(RepeatedDemand::Scheduled(vec![ + TimestampMs(10), + TimestampMs(20), + ])), + Some((6, 3)), + ), + entry( + QueryRecurrence::OneTime { + invocations: 3, + execute_at: None, + }, + Some((8, 2)), + ), + entry(QueryRecurrence::Unknown, Some((10, 1))), + ]); + } + #[test] + fn same_step_large_ranges_are_counted_without_timestamp_enumeration() { + let entries = [ + fixed(1000, None, Some((1_000_000_000, 1))), + fixed(1000, None, Some((500_000_000, 1))), + ]; + let schedule = QueryScheduleRates::new(&entries, 60.0, None).unwrap(); + assert_eq!(schedule.evaluations.fixed[0].unknown_atoms.len(), 2); + assert_eq!( + schedule.union_rate([true, true].into_iter()), + 1_000_000_001.0 + ); + assert_eq!(schedule.request_union_rate([true, true].into_iter()), 1.0); + } + #[test] + fn large_coprime_ranges_use_exact_arithmetic_coverage() { + let entries = [ + fixed(1000, None, Some((1_000_000_000, 2))), + fixed(1000, None, Some((1_000_000_000, 3))), + ]; + let schedule = QueryScheduleRates::new(&entries, 60.0, None).unwrap(); + assert_eq!(schedule.evaluations.fixed[0].unknown_atoms.len(), 3); + assert_eq!(schedule.union_rate([true, true].into_iter()), 666_666_668.0); + assert_eq!( + schedule.union_rate([true, false].into_iter()), + 500_000_001.0 + ); + assert_eq!( + schedule.union_rate([false, true].into_iter()), + 333_333_334.0 + ); + compare_all_subsets(&[ + fixed(1000, None, Some((60, 2))), + fixed(1000, None, Some((60, 3))), + ]); + } + #[test] + fn arithmetic_atoms_match_offset_coverage_with_duplicates_and_lcm_overflow() { + let readers = [(0, 2, 60), (1, 3, 60), (2, 2, 60), (3, 5, 37), (4, 7, 0)]; + let atoms = coverage(&readers, None).unwrap(); + let actual: BTreeMap<_, _> = atoms + .into_iter() + .map(|atom| (atom.readers, atom.count)) + .collect(); + let mut expected = BTreeMap::, u64>::new(); + for offset in 0..=60 { + let members: Vec<_> = readers + .iter() + .filter(|(_, step, span)| offset <= *span && offset % step == 0) + .map(|(id, _, _)| *id) + .collect(); + if !members.is_empty() { + *expected.entry(members).or_default() += 1; + } + } + assert_eq!(actual, expected); + let large = 1u64 << 63; + let atoms = coverage(&[(0, large - 1, large), (1, large - 3, large)], None).unwrap(); + assert_eq!(atoms.iter().map(|atom| atom.count).sum::(), 3); + assert!(atoms + .iter() + .any(|atom| atom.readers == vec![0, 1] && atom.count == 1)); + } + #[test] + fn excessive_mixed_offset_domains_and_deadlines_are_explicit() { + let entries: Vec<_> = (1..=13) + .map(|step| fixed(1000, None, Some((10_000_000, step)))) + .collect(); + assert!(matches!( + QueryScheduleRates::new(&entries, 60.0, None), + Err(CandidateSelectionError::UnsupportedDirectCost(_)) + )); + assert!(matches!( + QueryScheduleRates::new(&entries, 60.0, Some(Instant::now())), + Err(CandidateSelectionError::Solver(_)) + )); + } +} diff --git a/crates/plan-selection/src/search/milp/general_schedule_tests.rs b/crates/plan-selection/src/search/milp/general_schedule_tests.rs new file mode 100644 index 000000000..0305324ad --- /dev/null +++ b/crates/plan-selection/src/search/milp/general_schedule_tests.rs @@ -0,0 +1,254 @@ +use super::*; +use crate::search::milp::model::{solve_direct_cost_proposal_with_metrics, CandidateSolveMetrics}; +use asap_types::ir::properties::AccuracyTarget; +use asap_types::workload::{ + DemandEstimate, Predictability, Query, QueryRecurrence, QueryRequirements, QueryWorkloadEntry, + RangeEvaluation, Rate, RepeatedDemand, RepetitionInterval, +}; + +fn entry(recurrence: QueryRecurrence, evaluations: u64) -> QueryWorkloadEntry { + QueryWorkloadEntry { + query: Query("sum(m)".into()), + requirements: QueryRequirements { + accuracy: AccuracyTarget::exact(), + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence, + range_evaluation: (evaluations > 1).then_some(RangeEvaluation { + span_ms: (evaluations - 1) * 1000, + step_ms: 1000, + }), + } +} +fn fixed(interval: u32, evaluations: u64) -> QueryWorkloadEntry { + entry( + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(interval))), + evaluations, + ) +} +fn node(id: usize, input: Option) -> PhysicalCandidate { + PhysicalCandidate { + group: asap_logical_asap_optimization::memo::MemoGroupId(id as u32), + local_cost: 0.0, + producer_requirements: input + .into_iter() + .map(|input| ProducerRequirement { + compatible_producers: vec![PhysicalCandidateId(input)].into(), + }) + .collect(), + } +} +fn check_cost( + entries: &[QueryWorkloadEntry], + expected_rates: &[f64], + expected_union: f64, + per_request: bool, +) { + let general = QueryScheduleRates::new(entries, 60.0, None).unwrap(); + for (actual, expected) in general.per_query_rates().iter().zip(expected_rates) { + assert!((actual - expected).abs() < 1e-12); + } + let schedule = general; + let mut candidates = vec![node(0, None), node(1, Some(0))]; + candidates.extend((0..entries.len()).map(|q| node(q + 2, Some(1)))); + let graph = CandidateDependencyGraph { + candidates, + query_roots: (0..entries.len()) + .map(|q| QueryRootCandidates { + compatible_candidates: vec![PhysicalCandidateId(q + 2)], + }) + .collect(), + ..Default::default() + }; + let mut nodes = vec![ + NodeCostProfile { + execution_cost: 2.0, + work_seconds: 1.0, + held_memory_cost_per_second: 3.0, + ..Default::default() + }, + NodeCostProfile { + execution_cost: 4.0, + work_seconds: 2.0, + held_memory_cost_per_second: if per_request { 0.0 } else { 2.0 }, + standing_cost_per_second: if per_request { 13.0 } else { 0.0 }, + materialized: true, + per_request, + ..Default::default() + }, + ]; + nodes.extend((0..entries.len()).map(|q| NodeCostProfile { + execution_cost: 1.0, + work_seconds: q as f64 + 1.0, + ..Default::default() + })); + let sum: f64 = expected_rates.iter().sum(); + let weighted: f64 = expected_rates + .iter() + .enumerate() + .map(|(q, rate)| rate * (q as f64 + 4.0)) + .sum(); + let longest = entries.len() as f64 + 3.0; + let expected = expected_union + * (6.0 + 3.0 * weighted / sum + if per_request { 0.0 } else { 2.0 * longest }) + + sum + + if per_request { 13.0 } else { 0.0 }; + let profile = DirectCostProfile { + nodes, + schedules: Some(schedule), + query_rate: 1.0, + request_latency_seconds: 0.0, + request_cost: 0.0, + }; + let mut metrics = CandidateSolveMetrics::default(); + let proposal = + solve_direct_cost_proposal_with_metrics(&graph, &[], None, &mut metrics, &profile) + .unwrap() + .unwrap(); + let direct = proposal.direct_modeled_cost.unwrap(); + assert!( + (direct - expected).abs() < 1e-8 * expected.max(1.0), + "direct {direct}, independent expected {expected}" + ); + assert!((proposal.model_lower_bound.unwrap() - expected).abs() < 1e-6 * expected.max(1.0)); + assert!( + metrics.peak_variables < 5000, + "a rate lattice or reader subset product is unnecessary" + ); +} + +#[test] +fn noncommensurate_fixed_intervals_price_weighted_memory_without_integer_lattice() { + let intervals = [59_000, 61_000, 67_000, 71_000]; + let entries: Vec<_> = intervals + .iter() + .map(|interval| fixed(*interval, 1)) + .collect(); + let rates: Vec<_> = intervals + .iter() + .map(|interval| 1000.0 / f64::from(*interval)) + .collect(); + check_cost(&entries, &rates, rates.iter().sum(), false); +} + +#[test] +fn mixed_schedules_price_materialized_union_and_unmaterialized_mean() { + let estimate = QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(0.137), + observed_at: None, + valid_for: None, + })); + let entries = vec![ + fixed(1000, 2), + fixed(2000, 3), + entry(estimate, 4), + entry( + QueryRecurrence::OneTime { + invocations: 3, + execute_at: None, + }, + 5, + ), + entry(QueryRecurrence::Unknown, 6), + ]; + let rates = [2.0, 1.5, 0.548, 0.25, 0.1]; + // Fixed intervals and estimates add; the two batch demands use max(15,6)/60. + check_cost(&entries, &rates, 2.0 + 1.5 + 0.548 + 0.25, false); + // Retained refreshes use request rates; readers and weighted lifetime still + // use evaluation rates including each query's different range length. + check_cost(&entries, &rates, 1.0 + 0.5 + 0.137 + 3.0 / 60.0, true); +} + +#[test] +fn zero_rate_readers_preserve_materialized_and_unmaterialized_lifetimes() { + // With all rates zero the execution cost is zero. With one positive reader, + // unmaterialized lifetime ignores the zero-rate reader, while a shared + // query-time materialization still uses its longest reached latency. + for rates in [[0.0, 0.0], [0.0, 0.5], [0.5, 0.0]] { + let entries: Vec<_> = rates + .iter() + .map(|rate| { + entry( + QueryRecurrence::Repeated(RepeatedDemand::EstimatedRate(DemandEstimate { + expected_rate: Rate(*rate), + observed_at: None, + valid_for: None, + })), + 1, + ) + }) + .collect(); + let general = QueryScheduleRates::new(&entries, 60.0, None).unwrap(); + let graph = CandidateDependencyGraph { + candidates: vec![ + node(0, None), + node(1, Some(0)), + node(2, Some(1)), + node(3, Some(1)), + ], + query_roots: vec![ + QueryRootCandidates { + compatible_candidates: vec![PhysicalCandidateId(2)], + }, + QueryRootCandidates { + compatible_candidates: vec![PhysicalCandidateId(3)], + }, + ], + ..Default::default() + }; + let profile = DirectCostProfile { + nodes: vec![ + NodeCostProfile { + execution_cost: 2.0, + work_seconds: 1.0, + held_memory_cost_per_second: 3.0, + ..Default::default() + }, + NodeCostProfile { + materialized: true, + execution_cost: 4.0, + work_seconds: 2.0, + held_memory_cost_per_second: 2.0, + ..Default::default() + }, + NodeCostProfile { + execution_cost: 1.0, + work_seconds: 1.0, + ..Default::default() + }, + NodeCostProfile { + execution_cost: 1.0, + work_seconds: 2.0, + ..Default::default() + }, + ], + schedules: Some(general), + query_rate: 1.0, + request_latency_seconds: 0.0, + request_cost: 0.0, + }; + let sum: f64 = rates.iter().sum(); + let lifetime = if sum > 0.0 { + (rates[0] * 4.0 + rates[1] * 5.0) / sum + } else { + 5.0 + }; + let expected = sum * (2.0 + 4.0 + 3.0 * lifetime + 2.0 * 5.0) + sum; + let proposal = solve_direct_cost_proposal_with_metrics( + &graph, + &[], + None, + &mut Default::default(), + &profile, + ) + .unwrap() + .unwrap(); + assert!( + (proposal.direct_modeled_cost.unwrap() - expected).abs() < 1e-8, + "rates {rates:?}: expected {expected}, got {:?}", + proposal.direct_modeled_cost + ); + assert!((proposal.model_lower_bound.unwrap() - expected).abs() < 1e-6); + } +} diff --git a/crates/plan-selection/src/search/milp/model.rs b/crates/plan-selection/src/search/milp/model.rs index f3d78fde8..ca4aa32a4 100644 --- a/crates/plan-selection/src/search/milp/model.rs +++ b/crates/plan-selection/src/search/milp/model.rs @@ -812,6 +812,19 @@ pub(super) fn solve_with_objective( } } if let Some(projection) = projection { + projection.constrain_input_tuples( + &mut variables, + &candidate_selected, + &connections, + &mut constraints, + check_deadline, + )?; + projection.constrain_reader_contexts( + &candidate_selected, + &objective_reach, + &mut constraints, + check_deadline, + )?; projection.constrain_observations( &candidate_selected, &connections, diff --git a/crates/plan-selection/src/search/milp/schedule_selection_tests.rs b/crates/plan-selection/src/search/milp/schedule_selection_tests.rs new file mode 100644 index 000000000..16767b398 --- /dev/null +++ b/crates/plan-selection/src/search/milp/schedule_selection_tests.rs @@ -0,0 +1,355 @@ +use super::*; +use crate::search::milp::model::{solve_direct_cost_proposal_with_metrics, CandidateSolveMetrics}; +use asap_logical_asap_optimization::memo::MemoGroupId; +use asap_types::workload::{QueryRecurrence, RepeatedDemand, RepetitionInterval}; + +fn union(queries: &[QueryRecurrence], mask: usize) -> f64 { + crate::cost::execution_demand::evaluation_rate( + queries + .iter() + .enumerate() + .filter(|(q, _)| mask & (1 << q) != 0) + .map(|(_, query)| (query, None)), + 3600.0, + ) + .unwrap() +} +fn schedules(queries: &[QueryRecurrence]) -> QueryScheduleRates { + use asap_types::workload::{Predictability, Query, QueryRequirements, QueryWorkloadEntry}; + let entries: Vec<_> = queries + .iter() + .map(|recurrence| QueryWorkloadEntry { + query: Query("m".into()), + requirements: QueryRequirements { + accuracy: asap_types::ir::properties::AccuracyTarget::exact(), + max_latency_ms: None, + }, + predictability: Predictability::Predictable { known_at: None }, + recurrence: recurrence.clone(), + range_evaluation: None, + }) + .collect(); + QueryScheduleRates::new(&entries, 3600.0, None).unwrap() +} +pub(super) fn rate_schedule(rates: &[f64]) -> QueryScheduleRates { + schedules( + &rates + .iter() + .map(|rate| { + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval( + (1000.0 / rate) as u32, + ))) + }) + .collect::>(), + ) +} +fn node(input: Option) -> PhysicalCandidate { + PhysicalCandidate { + group: MemoGroupId(0), + local_cost: 0.0, + producer_requirements: input + .into_iter() + .map(|id| ProducerRequirement { + compatible_producers: vec![PhysicalCandidateId(id)].into(), + }) + .collect(), + } +} +fn root(id: usize) -> QueryRootCandidates { + QueryRootCandidates { + compatible_candidates: vec![PhysicalCandidateId(id)], + } +} + +// Boundary demand is the evaluator's union rate; upstream held memory uses +// the rate-weighted mean, not the arithmetic mean or the sum of root rates. +#[test] +fn heterogeneous_intervals_price_boundary_rates_and_weighted_lifetimes() { + let queries = [500, 1000] + .map(|ms| QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(ms)))); + let graph = CandidateDependencyGraph { + candidates: vec![node(None), node(Some(0)), node(Some(1)), node(Some(1))], + query_roots: vec![root(2), root(3)], + ..Default::default() + }; + let profile = DirectCostProfile { + schedules: Some(schedules(&queries)), + query_rate: 1.0, + request_latency_seconds: 0.5, + request_cost: 7.0, + nodes: vec![ + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 2.0, + work_seconds: 1.0, + compaction_seconds: 0.0, + held_memory_cost_per_second: 3.0, + ..Default::default() + }, + NodeCostProfile { + per_request: false, + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 1.0, + work_seconds: 2.0, + compaction_seconds: 0.0, + held_memory_cost_per_second: 4.0, + materialized: true, + raw_retention: None, + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + work_seconds: 3.0, + compaction_seconds: 0.0, + ..Default::default() + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + work_seconds: 4.0, + compaction_seconds: 0.0, + ..Default::default() + }, + ], + }; + assert_eq!( + profile + .schedules + .as_ref() + .unwrap() + .union_rate([true, true].into_iter()), + 3.0 + ); + let result = solve_direct_cost_proposal_with_metrics( + &graph, + &[], + None, + &mut CandidateSolveMetrics::default(), + &profile, + ) + .unwrap() + .unwrap(); + assert!((result.direct_modeled_cost.unwrap() - 167.5).abs() < 1e-8); + assert!((result.model_lower_bound.unwrap() - 167.5).abs() < 1e-8); +} + +// Same-interval readers can share one evaluation only at compatible phases. +// Compare with the evaluator's policy rather than assuming equal rates merge. +#[test] +fn aligned_and_offset_phases_use_evaluator_union_policy() { + use asap_types::workload::TimestampMs; + let graph = CandidateDependencyGraph { + candidates: vec![node(None)], + query_roots: vec![root(0), root(0)], + ..Default::default() + }; + for (second_phase, expected) in [(0, 7.0), (500, 14.0)] { + let queries = [0, second_phase].map(|phase| { + QueryRecurrence::Repeated(RepeatedDemand::FixedIntervalAt { + interval: RepetitionInterval(1000), + evaluation_phase: TimestampMs(phase), + }) + }); + let profile = DirectCostProfile { + schedules: Some(schedules(&queries)), + query_rate: 1.0, + request_latency_seconds: 0.0, + request_cost: 0.0, + nodes: vec![NodeCostProfile { + per_request: false, + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 3.0, + work_seconds: 2.0, + compaction_seconds: 0.0, + held_memory_cost_per_second: 2.0, + materialized: true, + raw_retention: None, + }], + }; + let result = solve_direct_cost_proposal_with_metrics( + &graph, + &[], + None, + &mut CandidateSolveMetrics::default(), + &profile, + ) + .unwrap() + .unwrap(); + assert!((result.direct_modeled_cost.unwrap() - expected).abs() < 1e-8); + // HiGHS epigraph feasibility is tolerance-based; the decoded selected + // cost above is recomputed independently and remains exact here. + assert!( + (result.model_lower_bound.unwrap() - expected).abs() <= 1e-7 * expected, + "bound={:?}, expected={expected}", + result.model_lower_bound + ); + } +} + +// Every reader can independently choose either materialized producer. +// Compare the joint optimum with four independently priced assignments. +#[test] +fn heterogeneous_selection_matches_exhaustive_reader_assignments() { + let queries = [500, 1000] + .map(|ms| QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(ms)))); + let schedule = schedules(&queries); + let mut graph = CandidateDependencyGraph { + candidates: vec![node(None), node(None), node(Some(0)), node(Some(0))], + query_roots: vec![root(2), root(3)], + ..Default::default() + }; + for id in [2, 3] { + graph.candidates[id].producer_requirements[0].compatible_producers = + vec![PhysicalCandidateId(0), PhysicalCandidateId(1)].into(); + } + let profile = DirectCostProfile { + schedules: Some(schedule.clone()), + query_rate: 1.0, + request_latency_seconds: 0.5, + request_cost: 7.0, + nodes: vec![ + NodeCostProfile { + per_request: false, + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 10.0, + work_seconds: 1.0, + compaction_seconds: 0.0, + held_memory_cost_per_second: 3.0, + materialized: true, + raw_retention: None, + }, + NodeCostProfile { + per_request: false, + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 2.0, + work_seconds: 0.5, + compaction_seconds: 0.0, + held_memory_cost_per_second: 1.0, + materialized: true, + raw_retention: None, + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 1.0, + work_seconds: 2.0, + compaction_seconds: 0.0, + ..Default::default() + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 1.0, + work_seconds: 3.0, + compaction_seconds: 0.0, + ..Default::default() + }, + ], + }; + let expected = [[0, 0], [0, 1], [1, 0], [1, 1]] + .into_iter() + .map(|assignment| { + let latency = [ + 2.5 + [1.0, 0.5][assignment[0]], + 3.5 + [1.0, 0.5][assignment[1]], + ]; + let mut cost = 7.0 + 3.0; + for supplier in [0, 1] { + let mask = assignment + .iter() + .enumerate() + .filter(|(_, chosen)| **chosen == supplier) + .fold(0, |mask, (q, _)| mask | (1 << q)); + let longest = latency + .iter() + .enumerate() + .filter(|(q, _)| mask & (1 << q) != 0) + .map(|(_, time)| *time) + .fold(0.0, f64::max); + cost += union(&queries, mask) + * ([10.0, 2.0][supplier] + [3.0, 1.0][supplier] * longest); + } + cost + }) + .fold(f64::INFINITY, f64::min); + let result = solve_direct_cost_proposal_with_metrics( + &graph, + &[], + None, + &mut CandidateSolveMetrics::default(), + &profile, + ) + .unwrap() + .unwrap(); + assert!((result.direct_modeled_cost.unwrap() - expected).abs() < 1e-8); + // HiGHS epigraph feasibility is tolerance-based; the decoded selected + // cost above is recomputed independently and remains exact here. + assert!( + (result.model_lower_bound.unwrap() - expected).abs() <= 1e-7 * expected, + "bound={:?}, expected={expected}", + result.model_lower_bound + ); +} +// A rare slow reader's weighted held lifetime can dominate cost despite a +// tiny rate. Reject that numerically unsupported mean before optimization, +// rather than erase its contribution in a relaxed equality. +#[test] +fn extreme_reader_rates_are_explicitly_unsupported() { + let queries = [1, u32::MAX] + .map(|ms| QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(ms)))); + let graph = CandidateDependencyGraph { + candidates: vec![node(None), node(None), node(Some(0)), node(Some(0))], + query_roots: vec![root(2), root(3)], + ..Default::default() + }; + let profile = DirectCostProfile { + schedules: Some(schedules(&queries)), + query_rate: 1.0, + request_latency_seconds: 0.0, + request_cost: 0.0, + nodes: vec![ + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 1.0, + held_memory_cost_per_second: 1e6, + ..Default::default() + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + execution_cost: 2.0, + held_memory_cost_per_second: 1e6, + ..Default::default() + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + work_seconds: 1.0, + compaction_seconds: 0.0, + ..Default::default() + }, + NodeCostProfile { + ingestion: false, + standing_cost_per_second: 0.0, + work_seconds: 1e9, + compaction_seconds: 0.0, + ..Default::default() + }, + ], + }; + let rates = &profile.schedules.as_ref().unwrap().per_query_rates(); + assert!( + rates[1] * 1e9 * 1e6 > 1e8, + "the rare reader has meaningful memory cost" + ); + assert!(matches!( + profile.validate(&graph), + Err(CandidateSelectionError::UnsupportedDirectCost(_)) + )); +} diff --git a/crates/plan-selection/src/search/refinement/planner.rs b/crates/plan-selection/src/search/refinement/planner.rs index 76cf52c9f..34c21a1af 100644 --- a/crates/plan-selection/src/search/refinement/planner.rs +++ b/crates/plan-selection/src/search/refinement/planner.rs @@ -138,13 +138,15 @@ pub(in crate::search) fn optimize_configured_candidates( &mut joint_fallback_reasons, )?; let mut direct = Vec::new(); - // Keep the older restricted profile only as a bounded fallback when the + // Use a fixed-interface profile as a bounded fallback when the // joint interface catalog cannot be certified within the planning budget. + // Both adapters share the same construction deadline, preserving time for + // the feasibility search even when direct pricing is unsupported. if joint.is_none() { - if let Some(seed) = direct_cost_seed(configured, context, &exclusions, false)? { + if let Some(seed) = direct_cost_seed(configured, &joint_context, &exclusions, false)? { direct.push(seed); } - if let Some(seed) = direct_cost_seed(configured, context, &exclusions, true)? { + if let Some(seed) = direct_cost_seed(configured, &joint_context, &exclusions, true)? { direct.push(seed); } } @@ -282,9 +284,11 @@ fn joint_cost_seed( let sizes: Vec<_> = profile.candidates.iter().map(Vec::len).collect(); for options in &mut profile.candidates { for option in options { - option - .compatible_suppliers - .retain(|binding| binding.price_option < sizes[binding.producer.0]); + option.compatible_input_tuples.retain(|tuple| { + tuple + .iter() + .all(|binding| binding.price_option < sizes[binding.producer.0]) + }); } } Ok(Some((configured.inventory.graph.clone(), profile))) @@ -315,7 +319,9 @@ fn direct_cost_seed( }; let mut extracted = match extracted { Ok(Some(extracted)) => extracted, - Ok(None) | Err(DirectProfileError::DeadlineExceeded(_)) => return Ok(None), + Ok(None) + | Err(DirectProfileError::DeadlineExceeded(_)) + | Err(DirectProfileError::Unsupported(_)) => return Ok(None), Err(error) => return Err(error.to_string()), }; if context @@ -1113,6 +1119,46 @@ mod tests { ) } + // An optional direct-price proposal must not abort planning when its + // schedule construction budget expires; the ordinary search remains usable. + #[test] + fn unsupported_schedule_profiles_leave_both_seed_fallbacks_available() { + let (configured, roots, mut entries, data) = fixture(); + entries[1].recurrence = QueryRecurrence::Unknown; + let context = CandidatePlanningContext { + deadline: Some(std::time::Instant::now()), + refine_materialization: false, + original_roots: &roots, + entries: &entries, + data: &data, + models: PlanningModels::builtin(), + calibration: CalibrationFit::default(), + }; + let mut reasons = Vec::new(); + assert!(joint_cost_seed(&configured, &context, &[], &mut reasons) + .unwrap() + .is_none()); + assert!(!reasons.is_empty()); + for latest_panes in [false, true] { + assert!(direct_cost_seed(&configured, &context, &[], latest_panes) + .unwrap() + .is_none()); + } + // Malformed demand is a real input error, not an unsupported model. + let mut invalid_entries = entries.clone(); + invalid_entries[1].recurrence = + QueryRecurrence::Repeated(RepeatedDemand::FixedInterval(RepetitionInterval(0))); + let invalid_context = CandidatePlanningContext { + deadline: None, + entries: &invalid_entries, + ..context + }; + assert!(joint_cost_seed(&configured, &invalid_context, &[], &mut reasons).is_err()); + for latest_panes in [false, true] { + assert!(direct_cost_seed(&configured, &invalid_context, &[], latest_panes).is_err()); + } + } + // One logical group can serve a repeated reader through panes and an // unpredictable reader locally; only selected pane consumers need eligibility. #[test] diff --git a/docs/design_docs/proposals/joint-planning-search.md b/docs/design_docs/proposals/joint-planning-search.md index c5f7a8c20..2ea1befbe 100644 --- a/docs/design_docs/proposals/joint-planning-search.md +++ b/docs/design_docs/proposals/joint-planning-search.md @@ -202,17 +202,21 @@ Memory-time and capacity answer different questions. A state occupying `m` bytes The direct-cost objective computes the total modeled cost of the selected execution DAG. It does not precompute one fixed total cost for each candidate or enumerate every complete DAG. Before solving, the planner computes each supported candidate's local cost parameters from its configuration and compatible suppliers. Inside MILP, the selected nodes and connections determine execution frequency and repeated work, consumer-specific operations, state holding costs, shared raw retention, and query overhead. A shared producer owns its charge once, including every required execution; its consumers' work is charged separately. **Local candidate cost parameters are precomputed; the complete selected DAG's cost is determined jointly by MILP.** -Direct pricing currently covers certified query-time exact and fixed sketch configurations, including KMV, KLL, DDS, Count-Min, CountSketch, Theta, and source-certified HLL. Capacity must already satisfy the applicable failure-probability sizing: a later resize would invalidate precomputed prices. Whole-result targets therefore add conservatively sized capacity candidates before physical costing, preserving the original candidates. Matching schedules use a compact demand model. Larger sets of fixed-interval schedules use phase activation and bounded integer rate totals, avoiding enumeration of every reader subset; rates follow the evaluator's accounting policy, including unknown phases. The normalized rate total is bounded at 4,096; small heterogeneous workloads can also use evaluator-derived subset tables. The model includes raw retention, request overhead, and latency-dependent temporary state, and checks the decoded total against the complete DAG evaluator. Accuracy still requires full output validation. +Direct pricing currently covers certified query-time exact and fixed sketch configurations, including KMV, KLL, DDS, Count-Min, CountSketch, Theta, and source-certified HLL. Capacity must already satisfy the applicable failure-probability sizing: a later resize would invalidate precomputed prices. Whole-result targets therefore add conservatively sized capacity candidates before physical costing, preserving the original candidates. Matching schedules use a compact demand model. Mixed fixed intervals, known/unknown phases, estimated rates, scheduled requests, one-off requests and range evaluations use finite demand atoms following the evaluator's accounting policy. Different fixed intervals add; an unknown phase joins the first selected known phase on its interval; estimates add; nonrepeating demand uses the largest selected batch. These rules describe the configured cost model, not a new runtime scheduler. The production encoding needs neither a query-subset table nor an integer rate lattice. There is one production mixed-schedule encoding; the former subset and lattice encoders are removed. Behavioral selection tests use the current encoder and compare against separately computed evaluator costs. The model includes raw retention, request overhead, and latency-dependent temporary state, and checks the decoded total against the complete DAG evaluator. Accuracy still requires full output validation. Direct pane pricing covers query-time rebuilding, eligible state retained across evaluations, and fixed-retention ingestion maintenance. Source-certified HLL panes use the same population and failure-probability checks as HLL whole-window builds. Rebuilt state pays for execution demand and temporary lifetime; retained state pays for resident panes and refreshes. Ingestion work is excluded from query latency; the maximum compaction delay among reached ingestion states contributes to query latency and temporary holding cost. -**Supplier-dependent prices are joint decisions.** For each supported unary candidate, costing derives finite local pricing options from its compatible suppliers' priced output interfaces. An option records local CPU/state costs, output statistics, upstream work and the resulting certified worker count. For example, two supplier configurations requiring 0.9 and 2.4 CPU-seconds per ingestion second produce one-worker and three-worker options; their compaction delays need not agree. A downstream reader can likewise have different merge/extraction costs for different state capacities. The optimizer retains these options rather than keeping only the first price class. +**Supplier-dependent prices are joint decisions.** For each supported candidate, costing derives finite local pricing options from complete ordered tuples of its suppliers' priced output interfaces. Executable binary operations and relational joins preserve input positions even when two positions use the same producer. Shared ingestion ancestors contribute their work once to a worker-count certificate. An option records local CPU/state costs, output statistics, upstream work and the resulting certified worker count. For example, two supplier configurations requiring 0.9 and 2.4 CPU-seconds per ingestion second produce one-worker and three-worker options; their compaction delays need not agree. A downstream reader can likewise have different merge/extraction costs for different state capacities. The optimizer retains these options rather than keeping only the first price class. -Let `price[c,k]` select pricing option `k` of candidate `c`. Require `Σ_k price[c,k] = selected[c]`, and permit a selected consumer/producer connection only with a compatible pair of pricing options. Thus all consumers of one shared candidate observe the **same** chosen interface. The implementation lifts these options into an internal MILP graph with mutual exclusion per original candidate, reuses the demand/lifetime cost formulation, and projects the result back to the original candidate DAG. Pricing options are not extra executions or extra copies of a shared state. Rejected assignments and reader/resource constraints retain their original physical identities. +Let `price[c,k]` select pricing option `k` of candidate `c`. Require `Σ_k price[c,k] = selected[c]`, and permit a selected consumer/producer connection only with a compatible pair of pricing options. For multiple inputs, a tuple selector chooses one complete certified combination and determines every input connection; separately compatible suppliers cannot form an unpriced combination. Thus all consumers of one shared candidate observe the **same** chosen interface. The implementation lifts these options into an internal MILP graph with mutual exclusion per original candidate, reuses the demand/lifetime cost formulation, and projects the result back to the original candidate DAG. Pricing options are not extra executions or extra copies of a shared state. Rejected assignments and reader/resource constraints retain their original physical identities. -Local interface generation deduplicates identical outputs, costs and upstream-work certificates; it does not enumerate complete workload DAGs. It currently covers the supported unary operators. Acyclic catalogs use dependency order; catalogs with potential cycles propagate local interfaces for at most the candidate count, sufficient for every legal unary path. Query-time upstream work is not part of an ingestion worker certificate, preventing irrelevant path histories from multiplying equivalent interfaces. Final DAG ordering and one pricing option per physical candidate exclude cyclic executions. All legal scan-lookback classes participate jointly, with exact context matching on each scan connection; sharing one wider scan across narrower consumers requires an additional maximum-demand witness and remains outside this encoding. Multi-input pricing or domains exceeding the finite interface/time budget use the existing validated fallback. Generation reserves time for fallback rather than consuming the entire planning budget. A worker count is derived from the selected upstream work, not an independently tunable cluster size. The bounded production seed uses the first interchangeable execution copy while retaining all its pricing options; the full search retains every copy. This is a restricted proposal, not a proof that additional copies are unnecessary. Reports distinguish joint price-option solves from fixed-profile direct solves. +Local interface generation deduplicates identical outputs, costs and upstream-work certificates; it does not enumerate complete workload DAGs. It covers supported single-input and multi-input operators and timing configurations. Acyclic catalogs use dependency order; catalogs with potential cycles propagate local interfaces for at most the candidate count, sufficient for every legal acyclic dependency depth. Query-time upstream work is not part of an ingestion worker certificate, preventing irrelevant path histories from multiplying equivalent interfaces. Final DAG ordering and one pricing option per physical candidate exclude cyclic executions. All legal scan-lookback classes participate jointly, with exact context matching on each scan connection; sharing one wider scan across narrower consumers requires an additional maximum-demand witness and remains outside this encoding. Uncertified operators or domains exceeding the finite interface/tuple/time budget use the existing validated fallback. Generation reserves time for fallback rather than consuming the entire planning budget. Joint and fixed-price extraction share that reserved deadline. Unsupported profiles, including schedule-atom construction timeouts, skip the optional direct proposal and continue through validated search; invalid profile errors remain fatal. A worker count is derived from the selected upstream work, not an independently tunable cluster size. The bounded production seed uses the first interchangeable execution copy while retaining all its pricing options; the full search retains every copy. This is a restricted proposal, not a proof that additional copies are unnecessary. Reports distinguish joint price-option solves from fixed-profile direct solves. -Matching range requests on the same fixed schedule are directly priced for rebuilt and ingestion-maintained state: evaluation frequency includes every range step, request overhead remains per request, and maintained panes retain the required historical span. Query-kept range state, differing range contexts, uncertified supplier interfaces, and schedules outside the bounded encodings still require evaluator refinement. Unsupported configurations remain in the full candidate catalog and currently use evaluator-based refinement; these are implementation limitations to remove, not the intended final architecture. The restricted subspace's optimum is not a full-catalog optimum. Reports expose direct-cost solve counts, validated costs, and fallback reasons so a successful fallback cannot be mistaken for direct-model coverage. +Range demand distinguishes evaluations from requests. Query-local state follows the union of evaluation offsets, while eligible query-kept state refreshes once per request. Pane pricing options carry retention-span tiers: a selected tier must cover every selected reader and have a reader witnessing that maximum. This prevents both undercharging long ranges and charging retention demanded only by unselected readers. Existing pane alignment, freshness and materialization restrictions still apply. + +For a nonmaterialized node, let `A` be its normalized average holding time, `r_q` binary query reach, `e_q` the evaluation rate, and `L_q` query latency. Encode `Σ e_q × r_q × A = Σ e_q × r_q × L_q` by linearizing each binary/continuous product. Multiplying `A` by each selected execution-demand atom is linearized similarly. This models finite nonnegative rates without enumerating denominators or reader subsets. When all reached readers have zero expected demand, holding time falls back to the longest reader latency, matching the evaluator. Materialized state uses the longest reached-query latency. + +Common-step ranges compress to nested coverage counts, even for very long spans. Mixed-step ranges group identical reader-coverage sets. Small geometry domains use arithmetic intersections and exact inclusion/exclusion counts, so large spans need no timestamp expansion; larger domains use bounded offset construction. Excessive domains, ill-conditioned rates, or exhausted construction budgets report a fallback rather than coarsening the schedule. The evaluator uses the same exact intersection-count primitive when validating large mixed ranges. Unsupported contexts remain in the complete candidate catalog for validated evaluator refinement. Restricted-subspace bounds are not full-catalog optimum certificates. Reports expose direct-cost solve counts, validated costs and fallback reasons. A candidate whose evaluator cost overflows the representable range is skipped with a `cost_not_representable` diagnostic so scalar-price search can still find a validated plan. This is an unknown cost, not proof of infeasibility: if such assignments were excluded, the search reports a conservative zero lower bound and cannot claim a full-domain optimum or infeasibility. Invalid settings and other evaluator errors remain errors.