From 5c965f62084e4e5cc8981eddfcb926bd49c7c983 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 8 May 2025 17:58:14 +0200 Subject: [PATCH 01/93] Add "@" prefix to computed labels (#815) * bump version * add @ prefix for computed labels on the http level * labels starting with @ are not allowed for new records * update CHANGELOG --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 5 + Cargo.lock | 6 +- Cargo.toml | 2 +- integration_tests/api/http_test.py | 2 +- reduct_base/src/batch.rs | 23 ++++- reductstore/src/api/entry/read_batched.rs | 117 ++++++++++++++++------ reductstore/src/api/entry/write_single.rs | 24 ++--- 8 files changed, 126 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32f4b7e53..b8b20b85a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -280,7 +280,7 @@ jobs: version: [ "latest", - "v1.14.0", + "v1.14.8", "v1.13.5", "v1.12.4", "v1.11.2", diff --git a/CHANGELOG.md b/CHANGELOG.md index d3afcd30e..e510c281d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + +### Changed + +- Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) + ## [1.15.0] - 2025-05-07 ### Added diff --git a/Cargo.lock b/Cargo.lock index 95c685eb8..b75bb5d0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2001,7 +2001,7 @@ dependencies = [ [[package]] name = "reduct-base" -version = "1.15.0" +version = "1.16.0" dependencies = [ "async-trait", "bytes", @@ -2019,7 +2019,7 @@ dependencies = [ [[package]] name = "reduct-macros" -version = "1.15.0" +version = "1.16.0" dependencies = [ "quote", "syn 2.0.101", @@ -2027,7 +2027,7 @@ dependencies = [ [[package]] name = "reductstore" -version = "1.15.0" +version = "1.16.0" dependencies = [ "assert_matches", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 8e95eb43e..bfca05127 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.15.0" +version = "1.16.0" authors = ["Alexey Timin ", "ReductSoftware UG "] edition = "2021" rust-version = "1.85.0" diff --git a/integration_tests/api/http_test.py b/integration_tests/api/http_test.py index 85bf9e190..3e17ccdcc 100644 --- a/integration_tests/api/http_test.py +++ b/integration_tests/api/http_test.py @@ -6,7 +6,7 @@ def test_api_version(base_url, session): resp = session.get(f"{base_url}/info") assert resp.status_code == 200 - assert resp.headers["x-reduct-api"] == "1.15" + assert resp.headers["x-reduct-api"] == "1.16" def test_cors_allows_first_allowed_origin(base_url, session): diff --git a/reduct_base/src/batch.rs b/reduct_base/src/batch.rs index ab8e219bc..5eca01f9c 100644 --- a/reduct_base/src/batch.rs +++ b/reduct_base/src/batch.rs @@ -1,4 +1,4 @@ -// Copyright 2023 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -47,6 +47,13 @@ pub fn parse_batched_header(header: &str) -> Result { let mut rest = rest.to_string(); while let Some(pair) = rest.split_once('=') { let (key, value) = pair; + let key = key.trim(); + if key.starts_with("@") { + return Err(unprocessable_entity!( + "Label names must not start with '@': reserved for computed labels", + )); + } + rest = if value.starts_with('\"') { let value = value[1..].to_string(); let (value, rest) = value @@ -59,7 +66,7 @@ pub fn parse_batched_header(header: &str) -> Result { labels.insert(key.trim().to_string(), value.trim().to_string()); rest.trim().to_string() } else { - labels.insert(key.trim().to_string(), value.trim().to_string()); + labels.insert(key.to_string(), value.trim().to_string()); break; }; } @@ -151,9 +158,19 @@ mod tests { #[case("xxx")] fn test_parse_header_bad_header(#[case] header: &str) { let err = parse_batched_header(header).err().unwrap(); + assert_eq!(err, unprocessable_entity!("Invalid batched header")); + } + + #[rstest] + fn test_parse_header_bad_label() { + let err = parse_batched_header("123, text/plain, @label1=value1, label2=value2") + .err() + .unwrap(); assert_eq!( err, - ReductError::unprocessable_entity("Invalid batched header") + unprocessable_entity!( + "Label names must not start with '@': reserved for computed labels" + ) ); } } diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index caf739e23..f26d0472f 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -74,6 +74,38 @@ pub(crate) async fn read_batched_records( .await } +fn make_batch_header(reader: &BoxedReadRecord) -> (HeaderName, HeaderValue) { + let name = HeaderName::from_str(&format!("x-reduct-time-{}", reader.timestamp())).unwrap(); + let mut meta_data = vec![ + reader.content_length().to_string(), + reader.content_type().to_string(), + ]; + + let format_labels = |(k, v): (&String, &String)| { + if v.contains(",") { + format!("{}=\"{}\"", k, v) + } else { + format!("{}={}", k, v) + } + }; + + let mut labels: Vec = reader.labels().iter().map(format_labels).collect(); + + labels.extend( + reader + .computed_labels() + .iter() + .map(|(k, v)| format_labels((&format!("@{}", k), v))), + ); + + labels.sort(); + + meta_data.append(&mut labels); + let value: HeaderValue = meta_data.join(",").parse().unwrap(); + + (name, value) +} + async fn fetch_and_response_batched_records( bucket: Arc, entry_name: &str, @@ -82,33 +114,6 @@ async fn fetch_and_response_batched_records( io_settings: &IoConfig, ext_repository: &BoxedManageExtensions, ) -> Result { - let make_header = |reader: &BoxedReadRecord| { - let name = HeaderName::from_str(&format!("x-reduct-time-{}", reader.timestamp())).unwrap(); - let mut meta_data = vec![ - reader.content_length().to_string(), - reader.content_type().to_string(), - ]; - - let format_labels = |(k, v): (&String, &String)| { - if v.contains(",") { - format!("{}=\"{}\"", k, v) - } else { - format!("{}={}", k, v) - } - }; - - let mut labels: Vec = reader.labels().iter().map(format_labels).collect(); - - labels.extend(reader.computed_labels().iter().map(format_labels)); - - labels.sort(); - - meta_data.append(&mut labels); - let value: HeaderValue = meta_data.join(",").parse().unwrap(); - - (name, value) - }; - let mut header_size = 0usize; let mut body_size = 0u64; let mut headers = HeaderMap::new(); @@ -142,7 +147,7 @@ async fn fetch_and_response_batched_records( match reader { Ok(reader) => { { - let (name, value) = make_header(&reader); + let (name, value) = make_batch_header(&reader); header_size += name.as_str().len() + value.to_str().unwrap().len() + 2; body_size += reader.content_length(); headers.insert(name, value); @@ -154,8 +159,6 @@ async fn fetch_and_response_batched_records( || readers.len() > io_settings.batch_max_records || start_time.elapsed() > io_settings.batch_timeout { - // This is not correct, because we should check sizes before adding the record - // but we can't know the size in advance and after next() we can't go back break; } } @@ -269,15 +272,18 @@ impl Stream for ReadersWrapper { #[cfg(test)] mod tests { use super::*; - - use axum::body::to_bytes; + use async_trait::async_trait; use crate::api::entry::tests::query; + use axum::body::to_bytes; + use mockall::mock; use crate::api::tests::{components, headers, path_to_entry_1}; use crate::ext::ext_repository::create_ext_repository; use reduct_base::ext::ExtSettings; + use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::Labels; use rstest::*; use tempfile::tempdir; use tokio::time::sleep; @@ -489,6 +495,28 @@ mod tests { } } + #[rstest] + fn test_batch_compute_labels() { + let mut record = MockRecord::new(); + record.expect_timestamp().return_const(1000u64); + record.expect_content_length().return_const(100u64); + record + .expect_content_type() + .return_const("text/plain".to_string()); + record + .expect_labels() + .return_const(Labels::from_iter(vec![("a".to_string(), "b".to_string())])); + record + .expect_computed_labels() + .return_const(Labels::from_iter(vec![("x".to_string(), "y".to_string())])); + + let record: BoxedReadRecord = Box::new(record); + + let (name, value) = make_batch_header(&record); + assert_eq!(name, HeaderName::from_static("x-reduct-time-1000")); + assert_eq!(value.to_str().unwrap(), "100,text/plain,@x=y,a=b"); + } + #[fixture] fn ext_repository() -> BoxedManageExtensions { create_ext_repository(Some(tempdir().unwrap().into_path()), ExtSettings::default()).unwrap() @@ -506,4 +534,29 @@ mod tests { assert_eq!(wrapper.size_hint(), (0, None)); } } + + mock! { + Record {} + + impl RecordMeta for Record { + fn timestamp(&self) -> u64; + + fn labels(&self) -> &Labels; + } + + #[async_trait] + impl ReadRecord for Record { + async fn read(&mut self) -> Option>; + + fn content_length(&self) -> u64; + + fn content_type(&self) -> &str; + + fn last(&self) -> bool; + + fn computed_labels(&self) -> &Labels; + + fn computed_labels_mut(&mut self) -> &mut Labels; + } + } } diff --git a/reductstore/src/api/entry/write_single.rs b/reductstore/src/api/entry/write_single.rs index 46326d90c..361e2b180 100644 --- a/reductstore/src/api/entry/write_single.rs +++ b/reductstore/src/api/entry/write_single.rs @@ -2,7 +2,7 @@ // Licensed under the Business Source License 1.1 use crate::api::middleware::check_permissions; -use crate::api::{Components, ErrorCode, HttpError}; +use crate::api::{Components, HttpError}; use crate::auth::policy::WriteAccessPolicy; use axum::body::Body; use axum::extract::{Path, Query, State}; @@ -15,7 +15,7 @@ use crate::storage::storage::IO_OPERATION_TIMEOUT; use futures_util::StreamExt; use log::{debug, error}; use reduct_base::error::ReductError; -use reduct_base::{bad_request, Labels}; +use reduct_base::{bad_request, unprocessable_entity, Labels}; use std::collections::HashMap; use std::sync::Arc; use tokio::time::timeout; @@ -55,10 +55,11 @@ pub(crate) async fn write_record( let value = match v.to_str() { Ok(value) => value.to_string(), Err(_) => { - return Err(HttpError::new( - ErrorCode::UnprocessableEntity, - &format!("Label values for {} must be valid UTF-8 strings", k), - )); + return Err(unprocessable_entity!( + "Label values for {} must be valid UTF-8 strings", + k + ) + .into()); } }; labels.insert(key, value); @@ -141,6 +142,7 @@ mod tests { use axum_extra::headers::{Authorization, HeaderMapExt}; use reduct_base::io::RecordMeta; + use reduct_base::not_found; use rstest::*; #[rstest] @@ -211,10 +213,7 @@ mod tests { .err() .unwrap(); - assert_eq!( - err, - HttpError::new(ErrorCode::NotFound, "Bucket 'XXX' is not found") - ); + assert_eq!(err, not_found!("Bucket 'XXX' is not found").into()); } #[rstest] @@ -242,10 +241,7 @@ mod tests { assert_eq!( err, - HttpError::new( - ErrorCode::UnprocessableEntity, - "'ts' must be an unix timestamp in microseconds", - ) + unprocessable_entity!("'ts' must be an unix timestamp in microseconds").into() ); } From b8a5488dab25d2ea7dab21e64453876f5c7dac90 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 9 May 2025 07:31:31 +0200 Subject: [PATCH 02/93] Pass server information to extensions (#816) * pass server info to extensions * add api version variable * update CHANGELOG --- CHANGELOG.md | 4 ++ reduct_base/src/ext.rs | 2 + reduct_base/src/ext/ext_settings.rs | 51 ++++++++++++++++------- reductstore/src/api.rs | 21 ++++++---- reductstore/src/api/entry/read_batched.rs | 9 +++- reductstore/src/cfg.rs | 7 +++- reductstore/src/ext/ext_repository.rs | 10 ++++- 7 files changed, 78 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e510c281d..2f5fe43a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Pass server information to extensions, [PR-816](https://github.com/reductstore/reductstore/pull/816) + ### Changed - Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) diff --git a/reduct_base/src/ext.rs b/reduct_base/src/ext.rs index 030f0b3d6..0d465eb0e 100644 --- a/reduct_base/src/ext.rs +++ b/reduct_base/src/ext.rs @@ -19,6 +19,8 @@ pub use process_status::ProcessStatus; pub use ext_settings::{ExtSettings, ExtSettingsBuilder}; pub type BoxedReadRecord = Box; +pub const EXTENSION_API_VERSION: &str = "0.2"; + /// The trait for the IO extension. /// /// This trait is used to register queries and process records in a pipeline of extensions. diff --git a/reduct_base/src/ext/ext_settings.rs b/reduct_base/src/ext/ext_settings.rs index 3160d85a8..f6afbb204 100644 --- a/reduct_base/src/ext/ext_settings.rs +++ b/reduct_base/src/ext/ext_settings.rs @@ -3,41 +3,52 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use crate::msg::server_api::ServerInfo; + /// Settings for initializing an extension. #[derive(Debug, PartialEq, Clone)] pub struct ExtSettings { /// The log level for the extension. log_level: String, -} - -impl Default for ExtSettings { - fn default() -> Self { - Self { - log_level: "INFO".to_string(), - } - } + server_info: ServerInfo, } /// Builder for `ExtSettings`. pub struct ExtSettingsBuilder { - settings: ExtSettings, + log_level: String, + server_info: Option, } impl ExtSettingsBuilder { /// Creates a new `ExtSettingsBuilder`. fn new() -> Self { Self { - settings: ExtSettings::default(), + log_level: "INFO".to_string(), + server_info: None, } } /// Sets the log level for the extension. pub fn log_level(mut self, log_level: &str) -> Self { - self.settings.log_level = log_level.to_string(); + self.log_level = log_level.to_string(); + self + } + + /// Sets the server info for the extension. + pub fn server_info(mut self, server_info: ServerInfo) -> Self { + self.server_info = Some(server_info); self } + /// Builds the `ExtSettings`. + /// + /// # Panics + /// + /// Panics if the server info is not set. pub fn build(self) -> ExtSettings { - self.settings + ExtSettings { + log_level: self.log_level, + server_info: self.server_info.expect("Server info must be set"), + } } } @@ -47,6 +58,11 @@ impl ExtSettings { &self.log_level } + /// Returns the server info for the extension. + pub fn server_info(&self) -> &ServerInfo { + &self.server_info + } + /// Creates a new `ExtSettingsBuilder`. pub fn builder() -> ExtSettingsBuilder { ExtSettingsBuilder::new() @@ -59,8 +75,15 @@ mod tests { #[test] fn test_ext_settings_builder() { - let settings = ExtSettings::builder().log_level("info").build(); + let settings = ExtSettings::builder() + .log_level("INFO") + .server_info(ServerInfo { + version: "1.0".to_string(), + ..ServerInfo::default() + }) + .build(); - assert_eq!(settings.log_level(), "info"); + assert_eq!(settings.log_level(), "INFO"); + assert_eq!(settings.server_info().version, "1.0"); } } diff --git a/reductstore/src/api.rs b/reductstore/src/api.rs index b07117b92..8dbab465d 100644 --- a/reductstore/src/api.rs +++ b/reductstore/src/api.rs @@ -159,6 +159,11 @@ fn configure_cors(cors_allow_origin: &Vec) -> CorsLayer { #[cfg(test)] mod tests { + use crate::asset::asset_manager::create_asset_manager; + use crate::auth::token_repository::create_token_repository; + use crate::cfg::replication::ReplicationConfig; + use crate::ext::ext_repository::create_ext_repository; + use crate::replication::create_replication_repo; use axum::body::Body; use axum::extract::Path; use axum_extra::headers::{Authorization, HeaderMap, HeaderMapExt}; @@ -166,16 +171,11 @@ mod tests { use reduct_base::ext::ExtSettings; use reduct_base::msg::bucket_api::BucketSettings; use reduct_base::msg::replication_api::ReplicationSettings; + use reduct_base::msg::server_api::ServerInfo; use reduct_base::msg::token_api::Permissions; use rstest::fixture; use std::collections::HashMap; - use crate::asset::asset_manager::create_asset_manager; - use crate::auth::token_repository::create_token_repository; - use crate::cfg::replication::ReplicationConfig; - use crate::ext::ext_repository::create_ext_repository; - use crate::replication::create_replication_repo; - use super::*; mod http_error { @@ -283,8 +283,13 @@ mod tests { base_path: "/".to_string(), replication_repo: RwLock::new(replication_repo), io_settings: IoConfig::default(), - ext_repo: create_ext_repository(None, ExtSettings::default()) - .expect("Failed to create extension repo"), + ext_repo: create_ext_repository( + None, + ExtSettings::builder() + .server_info(ServerInfo::default()) + .build(), + ) + .expect("Failed to create extension repo"), }; Arc::new(components) diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index f26d0472f..55d7d0aff 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -283,6 +283,7 @@ mod tests { use crate::ext::ext_repository::create_ext_repository; use reduct_base::ext::ExtSettings; use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::msg::server_api::ServerInfo; use reduct_base::Labels; use rstest::*; use tempfile::tempdir; @@ -519,7 +520,13 @@ mod tests { #[fixture] fn ext_repository() -> BoxedManageExtensions { - create_ext_repository(Some(tempdir().unwrap().into_path()), ExtSettings::default()).unwrap() + create_ext_repository( + Some(tempdir().unwrap().into_path()), + ExtSettings::builder() + .server_info(ServerInfo::default()) + .build(), + ) + .unwrap() } mod stream_wrapper { diff --git a/reductstore/src/cfg.rs b/reductstore/src/cfg.rs index 8befea23c..d4b47d5d8 100644 --- a/reductstore/src/cfg.rs +++ b/reductstore/src/cfg.rs @@ -90,6 +90,8 @@ impl Cfg { None }; + let server_info = storage.info()?; + Ok(Components { storage, token_repo: tokio::sync::RwLock::new(token_repo), @@ -98,7 +100,10 @@ impl Cfg { replication_repo: tokio::sync::RwLock::new(replication_engine), ext_repo: create_ext_repository( ext_path, - ExtSettings::builder().log_level(&self.log_level).build(), + ExtSettings::builder() + .log_level(&self.log_level) + .server_info(server_info) + .build(), )?, base_path: self.api_base_path.clone(), diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index 660137eaf..d212649fb 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -335,6 +335,7 @@ pub(super) mod tests { use mockall::predicate::eq; use prost_wkt_types::Timestamp; use reduct_base::io::{ReadChunk, ReadRecord, RecordMeta}; + use reduct_base::msg::server_api::ServerInfo; use serde_json::json; use std::fs; use tempfile::tempdir; @@ -342,6 +343,7 @@ pub(super) mod tests { mod load { use super::*; + use reduct_base::msg::server_api::ServerInfo; #[log_test(rstest)] fn test_load_extension(ext_repo: ExtRepository) { assert_eq!(ext_repo.extension_map.len(), 1); @@ -381,7 +383,9 @@ pub(super) mod tests { #[fixture] fn ext_settings() -> ExtSettings { - ExtSettings::default() + ExtSettings::builder() + .server_info(ServerInfo::default()) + .build() } #[fixture] @@ -819,7 +823,9 @@ pub(super) mod tests { } fn mocked_ext_repo(name: &str, mock_ext: MockIoExtension) -> ExtRepository { - let ext_settings = ExtSettings::default(); + let ext_settings = ExtSettings::builder() + .server_info(ServerInfo::default()) + .build(); let mut ext_repo = ExtRepository::try_load(&tempdir().unwrap().into_path(), ext_settings).unwrap(); ext_repo.extension_map.insert( From 9665de3beae925cb82487f4667547998800e6390 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 09:12:14 +0200 Subject: [PATCH 03/93] Bump tempfile from 3.19.1 to 3.20.0 (#820) Bumps [tempfile](https://github.com/Stebalien/tempfile) from 3.19.1 to 3.20.0. - [Changelog](https://github.com/Stebalien/tempfile/blob/master/CHANGELOG.md) - [Commits](https://github.com/Stebalien/tempfile/compare/v3.19.1...v3.20.0) --- updated-dependencies: - dependency-name: tempfile dependency-version: 3.20.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b75bb5d0b..402b7dd63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2564,9 +2564,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", "getrandom 0.3.2", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 53203436e..281f9fa39 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -31,7 +31,7 @@ reduct-macros = { path = "../reduct_macros", version = "1.15.0" } chrono = { version = "0.4.41", features = ["serde"] } zip = "2.6.1" -tempfile = "3.19.1" +tempfile = "3.20.0" hex = "0.4.3" prost = "0.13.1" prost-wkt-types = "0.6.1" From 0d6b092666cc6b898890e59dcf4402b65d463831 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 09:13:11 +0200 Subject: [PATCH 04/93] Bump tokio from 1.44.2 to 1.45.0 (#819) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.44.2 to 1.45.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.44.2...tokio-1.45.0) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.45.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reduct_base/Cargo.toml | 2 +- reductstore/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 402b7dd63..4538ae25a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2701,9 +2701,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 4b53edd6c..3c9d10c9a 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -34,7 +34,7 @@ url = "2.5.4" http = "1.2.0" bytes = "1.10.0" async-trait = { version = "0.1.87" , optional = true } -tokio = { version = "1.44.2", optional = true, features = ["default", "rt", "time"] } +tokio = { version = "1.45.0", optional = true, features = ["default", "rt", "time"] } log = "0.4.0" thread-id = "5.0.0" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 281f9fa39..ada7a4c29 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -42,7 +42,7 @@ regex = "1.11.1" bytes = "1.10.1" axum = { version = "0.8.4", features = ["default", "macros"] } axum-extra = { version = "0.10.0", features = ["default", "typed-header"] } -tokio = { version = "1.44.2", features = ["full"] } +tokio = { version = "1.45.0", features = ["full"] } hyper = { version = "1.6.0", features = ["full"] } futures-util = "0.3.31" axum-server = { version = "0.7.1", features = ["tls-rustls"] } From 70ddc9dba603235beab9d8386bd522f2c8e4f9d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 09:13:31 +0200 Subject: [PATCH 05/93] Bump tower-http from 0.6.2 to 0.6.4 (#817) Bumps [tower-http](https://github.com/tower-rs/tower-http) from 0.6.2 to 0.6.4. - [Release notes](https://github.com/tower-rs/tower-http/releases) - [Commits](https://github.com/tower-rs/tower-http/compare/tower-http-0.6.2...tower-http-0.6.4) --- updated-dependencies: - dependency-name: tower-http dependency-version: 0.6.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4538ae25a..3bfede911 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2797,9 +2797,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" dependencies = [ "bitflags", "bytes", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index ada7a4c29..da93101ac 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -55,7 +55,7 @@ base64 = "0.22.1" ring = "0.17.12" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } async-stream = "0.3.6" -tower-http = { version = "0.6.2", features = ["cors"] } +tower-http = { version = "0.6.4", features = ["cors"] } crc64fast = "1.1.0" rustls = "0.23.26" byteorder = "1.5.0" From fdc136e1a77b590eb3b75b448ad96666d087787a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 09:13:42 +0200 Subject: [PATCH 06/93] Bump rustls from 0.23.26 to 0.23.27 (#818) Bumps [rustls](https://github.com/rustls/rustls) from 0.23.26 to 0.23.27. - [Release notes](https://github.com/rustls/rustls/releases) - [Changelog](https://github.com/rustls/rustls/blob/main/CHANGELOG.md) - [Commits](https://github.com/rustls/rustls/compare/v/0.23.26...v/0.23.27) --- updated-dependencies: - dependency-name: rustls dependency-version: 0.23.27 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- reductstore/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bfede911..06c7cf8ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2267,9 +2267,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.26" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "log", @@ -2301,9 +2301,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "aws-lc-rs", "ring", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index da93101ac..b6abee033 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -57,7 +57,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus async-stream = "0.3.6" tower-http = { version = "0.6.4", features = ["cors"] } crc64fast = "1.1.0" -rustls = "0.23.26" +rustls = "0.23.27" byteorder = "1.5.0" crossbeam-channel = "0.5.15" dlopen2="0.7.0" From e0a60a0380995270d8ddb0e492761246a765c1e0 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 15 May 2025 14:36:29 +0200 Subject: [PATCH 07/93] Integrate ReductSelect extension (#821) * embed reduct select extension * fix deprecated method * use sas url in ci/cd * use the rust version in docker * fix env * fix build * add sanity test for select extension * cover changes with tests * update CHANGELOG Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 16 ++- CHANGELOG.md | 1 + Dockerfile | 8 +- buildx.Dockerfile | 8 +- integration_tests/api/ext_test.py | 28 +++++ reductstore/Cargo.toml | 8 +- reductstore/build.rs | 66 +++++++++--- reductstore/src/api.rs | 3 +- reductstore/src/api/entry/read_batched.rs | 3 +- reductstore/src/asset/asset_manager.rs | 100 +++++++++++++++-- reductstore/src/auth/token_auth.rs | 2 +- reductstore/src/auth/token_repository.rs | 2 +- reductstore/src/cfg.rs | 14 +++ reductstore/src/cfg/provision/bucket.rs | 2 +- reductstore/src/cfg/provision/replication.rs | 2 +- reductstore/src/cfg/provision/token.rs | 2 +- reductstore/src/core/file_cache.rs | 2 +- reductstore/src/ext/ext_repository.rs | 102 +++++++++++------- reductstore/src/license.rs | 4 +- reductstore/src/main.rs | 4 +- .../src/replication/replication_repository.rs | 2 +- .../src/replication/replication_sender.rs | 2 +- .../src/replication/replication_task.rs | 2 +- .../src/replication/transaction_log.rs | 2 +- reductstore/src/storage/block_manager.rs | 2 +- .../src/storage/block_manager/block_index.rs | 8 +- reductstore/src/storage/block_manager/wal.rs | 2 +- reductstore/src/storage/bucket.rs | 2 +- reductstore/src/storage/entry.rs | 2 +- .../src/storage/entry/io/record_reader.rs | 2 +- .../src/storage/entry/io/record_writer.rs | 2 +- reductstore/src/storage/query.rs | 2 +- reductstore/src/storage/query/base.rs | 2 +- reductstore/src/storage/storage.rs | 2 +- 34 files changed, 312 insertions(+), 99 deletions(-) create mode 100644 integration_tests/api/ext_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8b20b85a..4db9720a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,6 @@ on: env: REGISTRY_IMAGE: reduct/store - MINIMAL_RUST_VERSION: 1.85.0 jobs: rust_fmt: @@ -45,6 +44,9 @@ jobs: context: . tags: ${{github.repository}}:latest outputs: type=docker,dest=/tmp/image.tar + build-args: | + ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} + RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} - name: Upload artifact uses: actions/upload-artifact@v4 @@ -94,7 +96,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: ${{ env.MINIMAL_RUST_VERSION }} + toolchain: ${{ vars.MINIMAL_RUST_VERSION }} - name: Install toolchain run: rustup target add ${{ matrix.target }} @@ -118,7 +120,8 @@ jobs: - name: Build binary env: RUSTFLAGS: "-C target-feature=+crt-static" - run: cargo build --release -p reductstore --target ${{ matrix.target }} + ARTIFACT_SAS_URL: ${{ secrets.ARTIFACT_SAS_URL }} + run: cargo build --release -p reductstore --target ${{ matrix.target }} --all-features - name: Upload binary uses: actions/upload-artifact@v4 @@ -199,6 +202,8 @@ jobs: repo-token: ${{ secrets.ACTION_GITHUB_TOKEN }} - name: Generate code coverage + env: + ARTIFACT_SAS_URL: ${{ secrets.ARTIFACT_SAS_URL }} run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info - name: Upload coverage to Codecov @@ -727,6 +732,9 @@ jobs: GIT_COMMIT=${{env.GITHUB_SHA}} CARGO_TARGET=${{matrix.cargo_target}} GCC_COMPILER=${{matrix.gcc_compiler}} + ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} + RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Export digest @@ -795,7 +803,7 @@ jobs: - name: Set up Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: ${{ env.MINIMAL_RUST_VERSION }} + toolchain: ${{ vars.MINIMAL_RUST_VERSION }} - uses: arduino/setup-protoc@v1 with: version: "3.x" diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f5fe43a8..aab8ded2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Pass server information to extensions, [PR-816](https://github.com/reductstore/reductstore/pull/816) +- Integrate ReductSelect v0.1.0, [PR-821](https://github.com/reductstore/reductstore/pull/821) ### Changed diff --git a/Dockerfile b/Dockerfile index c90c617db..15f78dc10 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,9 @@ FROM ubuntu:22.04 AS builder +ARG GIT_COMMIT=unspecified +ARG ARTIFACT_SAS_URL +ARG RUST_VERSION + RUN apt-get update && apt-get install -y \ cmake \ build-essential \ @@ -10,6 +14,7 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y # Add .cargo/bin to PATH ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustup default ${RUST_VERSION} WORKDIR /src @@ -20,8 +25,7 @@ COPY .cargo .cargo COPY Cargo.toml Cargo.toml COPY Cargo.lock Cargo.lock -ARG GIT_COMMIT=unspecified -RUN GIT_COMMIT=${GIT_COMMIT} cargo build --release +RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --all-features FROM ubuntu:22.04 diff --git a/buildx.Dockerfile b/buildx.Dockerfile index e8ec4fb99..373a1d3a8 100644 --- a/buildx.Dockerfile +++ b/buildx.Dockerfile @@ -7,6 +7,8 @@ ARG BUILDPLATFORM ARG CARGO_TARGET ARG GCC_COMPILER=gcc-11 ARG GIT_COMMIT=unspecified +ARG ARTIFACT_SAS_URL +ARG RUST_VERSION RUN apt-get update && apt-get install -y \ cmake \ @@ -17,9 +19,9 @@ RUN apt-get update && apt-get install -y \ ${GCC_COMPILER} RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - # Add .cargo/bin to PATH ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustup default ${RUST_VERSION} RUN rustup target add ${CARGO_TARGET} WORKDIR /src @@ -33,8 +35,8 @@ COPY Cargo.lock Cargo.lock RUN cargo install --force --locked bindgen-cli -RUN GIT_COMMIT=${GIT_COMMIT} cargo build --release --target ${CARGO_TARGET} --package reductstore -RUN cargo install reduct-cli --target ${CARGO_TARGET} --root /src/target/${CARGO_TARGET}/release +RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --target ${CARGO_TARGET} --package reductstore +RUN cargo install reduct-cli --target ${CARGO_TARGET} --root /src/target/${CARGO_TARGET}/release --all-features RUN mkdir /data diff --git a/integration_tests/api/ext_test.py b/integration_tests/api/ext_test.py new file mode 100644 index 000000000..141e39d98 --- /dev/null +++ b/integration_tests/api/ext_test.py @@ -0,0 +1,28 @@ +import json + +from integration_tests.api.conftest import requires_env + + +@requires_env("LICENSE_PATH") +def test__select_ext(base_url, bucket, session): + """Check if the select extension is available""" + resp = session.post(f"{base_url}/b/{bucket}/entry?ts=1", data="1,2,3,4,5") + assert resp.status_code == 200 + + resp = session.post( + f"{base_url}/b/{bucket}/entry/q", + json={ + "query_type": "QUERY", + "ext": {"select": {"columns": [{"index": 0, "as_label": "a"}]}}, + }, + ) + + assert resp.status_code == 200 + + query_id = int(json.loads(resp.content)["id"]) + assert query_id >= 0 + + resp = session.get(f"{base_url}/b/{bucket}/entry/batch?q={query_id}") + assert resp.status_code == 200 + + assert resp.headers["x-reduct-time-1"] == "2,application/octet-stream,@a=1" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index b6abee033..0af9ab6b8 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -19,6 +19,7 @@ include = ["src/**/*", "Cargo.toml", "Cargo.lock", "build.rs", "README.md", "LIC [features] default = ["web-console"] web-console = [] +select-ext = [] [lib] crate-type = ["lib"] @@ -33,7 +34,6 @@ chrono = { version = "0.4.41", features = ["serde"] } zip = "2.6.1" tempfile = "3.20.0" hex = "0.4.3" -prost = "0.13.1" prost-wkt-types = "0.6.1" rand = "0.9.1" serde = { version = "1.0.219", features = ["derive"] } @@ -60,13 +60,15 @@ crc64fast = "1.1.0" rustls = "0.23.27" byteorder = "1.5.0" crossbeam-channel = "0.5.15" -dlopen2="0.7.0" +dlopen2 = "0.7.0" log = "0.4" +prost = "0.13.1" [build-dependencies] prost-build = "0.13.1" -reqwest = { version = "0.12.15", default-features = false, features = ["rustls-tls", "blocking"] } +reqwest = { version = "0.12.15", default-features = false, features = ["rustls-tls", "blocking", "json"] } chrono = "0.4.41" +serde_json = "1.0.140" [dev-dependencies] mockall = "0.13.1" diff --git a/reductstore/build.rs b/reductstore/build.rs index c8a1a55a4..a366d7c67 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -1,9 +1,10 @@ // Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 -extern crate core; - -use reqwest::blocking::get; -use reqwest::StatusCode; +#[allow(unused_imports)] +use reqwest::{ + blocking::{get, Client}, + StatusCode, Url, +}; use std::path::Path; use std::time::SystemTime; use std::{env, fs}; @@ -25,7 +26,11 @@ fn main() -> Result<(), Box> { ) .expect("Failed to compile protos"); - download_web_console(); + #[cfg(feature = "web-console")] + download_web_console("v1.10.0"); + + #[cfg(feature = "select-ext")] + download_ext("select-ext", "v0.1.0"); // get build time and commit let build_time = chrono::DateTime::::from(SystemTime::now()) @@ -42,19 +47,17 @@ fn main() -> Result<(), Box> { println!("cargo:rustc-env=COMMIT={}", commit); Ok(()) } - #[cfg(feature = "web-console")] -fn download_web_console() { - const WEB_CONSOLE_VERSION: &str = "v1.10.0"; +fn download_web_console(version: &str) { let out_dir = env::var("OUT_DIR").unwrap(); - let console_path = &format!("{}/console-{}.zip", out_dir, WEB_CONSOLE_VERSION); + let console_path = &format!("{}/console-{}.zip", out_dir, version); if Path::exists(Path::new(console_path)) { return; } let mut resp = get(format!( "https://github.com/reductstore/web-console/releases/download/{}/web-console.build.zip", - WEB_CONSOLE_VERSION + version )) .expect("Failed to download Web Console"); if resp.status() != StatusCode::OK { @@ -69,5 +72,44 @@ fn download_web_console() { fs::copy(console_path, format!("{}/console.zip", out_dir)).expect("Failed to copy console.zip"); } -#[cfg(not(feature = "web-console"))] -fn download_web_console() {} +#[cfg(feature = "select-ext")] +fn download_ext(name: &str, version: &str) { + let sas_url = env::var("ARTIFACT_SAS_URL").unwrap_or_default(); + if sas_url.is_empty() { + panic!("ARTIFACT_SAS_URL is not set, disable the extensions feature"); + } + + let sas_url = Url::parse(&sas_url).expect("Failed to parse ARTIFACT_SAS_URL"); + + let target = env::var("TARGET").unwrap(); + let out_dir = env::var("OUT_DIR").unwrap(); + + let ext_path = &format!("{}/{}-{}.zip", out_dir.clone(), name, version); + if Path::exists(Path::new(ext_path)) { + return; + } + + println!("Downloading {}...", name); + let mut ext_url = sas_url + .join(&format!( + "/artifacts/{}/{}/{}.zip/{}.zip", + name, version, target, target + )) + .expect("Failed to create URL"); + ext_url.set_query(sas_url.query()); + + let client = Client::builder().user_agent("ReductStore").build().unwrap(); + let resp = client + .get(ext_url) + .send() + .expect(format!("Failed to download {}.zip", name).as_str()); + if resp.status() != StatusCode::OK { + panic!("Failed to download {}: {}", name, resp.status()); + } + + println!("Writing {}.zip...", ext_path); + + fs::write(ext_path, resp.bytes().unwrap()) + .expect(format!("Failed to write {}.zip", name).as_str()); + fs::copy(ext_path, format!("{}/{}.zip", out_dir, name)).expect("Failed to copy extension"); +} diff --git a/reductstore/src/api.rs b/reductstore/src/api.rs index 8dbab465d..7c22bdb41 100644 --- a/reductstore/src/api.rs +++ b/reductstore/src/api.rs @@ -220,7 +220,7 @@ mod tests { #[fixture] pub(crate) async fn components() -> Arc { - let data_path = tempfile::tempdir().unwrap().into_path(); + let data_path = tempfile::tempdir().unwrap().keep(); let storage = Storage::load(data_path.clone(), None); let mut token_repo = create_token_repository(data_path.clone(), "init-token"); @@ -285,6 +285,7 @@ mod tests { io_settings: IoConfig::default(), ext_repo: create_ext_repository( None, + vec![], ExtSettings::builder() .server_info(ServerInfo::default()) .build(), diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index 55d7d0aff..83cb129c4 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -521,7 +521,8 @@ mod tests { #[fixture] fn ext_repository() -> BoxedManageExtensions { create_ext_repository( - Some(tempdir().unwrap().into_path()), + Some(tempdir().unwrap().keep()), + vec![], ExtSettings::builder() .server_info(ServerInfo::default()) .build(), diff --git a/reductstore/src/asset/asset_manager.rs b/reductstore/src/asset/asset_manager.rs index 8a366e04b..2094e801b 100644 --- a/reductstore/src/asset/asset_manager.rs +++ b/reductstore/src/asset/asset_manager.rs @@ -5,6 +5,7 @@ use bytes::Bytes; use log::{debug, trace}; use std::fs::File; use std::io::{Cursor, Read}; +use std::path::PathBuf; use tempfile::{tempdir, TempDir}; use zip::ZipArchive; @@ -21,6 +22,9 @@ pub trait ManageStaticAsset { /// /// The file content as string. fn read(&self, relative_path: &str) -> Result; + + /// Get the absolute path of a file extracted from the zip archive. + fn absolut_path(&self, relative_path: &str) -> Result; } /// Asset manager that reads files from a zip archive as hex string and returns them as string @@ -95,6 +99,16 @@ impl ManageStaticAsset for ZipAssetManager { Ok(Bytes::from(content)) } + + fn absolut_path(&self, relative_path: &str) -> Result { + let path = self.path.path().join(relative_path); + if !path.try_exists()? { + return Err(ReductError::not_found( + format!("File {:?} not found", path).as_str(), + )); + } + Ok(path) + } } /// Empty asset manager that does not support any files @@ -104,6 +118,10 @@ impl ManageStaticAsset for NoAssetManager { fn read(&self, _relative_path: &str) -> Result { Err(ReductError::not_found("No static files supported")) } + + fn absolut_path(&self, _relative_path: &str) -> Result { + Err(ReductError::not_found("No static files supported")) + } } pub fn create_asset_manager(zipped_content: &[u8]) -> Box { @@ -117,13 +135,79 @@ pub fn create_asset_manager(zipped_content: &[u8]) -> Box PathBuf { + let archive = tempdir().unwrap().keep().join("test.zip"); + let mut file = File::create(archive.clone()).unwrap(); + file.write_all(b"test").unwrap(); + + // crete zip file + let mut zip = ZipWriter::new(file); + + zip.start_file::<&str, ExtendedFileOptions>("test.txt", FileOptions::default()) + .unwrap(); + zip.write_all(b"test").unwrap(); + zip.finish().unwrap(); + archive + } } } diff --git a/reductstore/src/auth/token_auth.rs b/reductstore/src/auth/token_auth.rs index bfb5daee6..62e75b814 100644 --- a/reductstore/src/auth/token_auth.rs +++ b/reductstore/src/auth/token_auth.rs @@ -91,7 +91,7 @@ mod tests { } fn setup() -> (Box, TokenAuthorization) { - let repo = create_token_repository(tempdir().unwrap().into_path(), "test"); + let repo = create_token_repository(tempdir().unwrap().keep(), "test"); let auth = TokenAuthorization::new("test"); (repo, auth) diff --git a/reductstore/src/auth/token_repository.rs b/reductstore/src/auth/token_repository.rs index 19cf79452..ad2a7877f 100644 --- a/reductstore/src/auth/token_repository.rs +++ b/reductstore/src/auth/token_repository.rs @@ -934,7 +934,7 @@ mod tests { #[fixture] fn path() -> PathBuf { - tempdir().unwrap().into_path() + tempdir().unwrap().keep() } #[fixture] diff --git a/reductstore/src/cfg.rs b/reductstore/src/cfg.rs index d4b47d5d8..9e090bfc6 100644 --- a/reductstore/src/cfg.rs +++ b/reductstore/src/cfg.rs @@ -80,6 +80,7 @@ impl Cfg { let storage = Arc::new(self.provision_buckets()); let token_repo = self.provision_tokens(); let console = create_asset_manager(load_console()); + let select_ext = create_asset_manager(load_select_ext()); let replication_engine = self.provision_replication_repo(Arc::clone(&storage))?; let ext_path = if let Some(ext_path) = &self.ext_path { @@ -100,6 +101,7 @@ impl Cfg { replication_repo: tokio::sync::RwLock::new(replication_engine), ext_repo: create_ext_repository( ext_path, + vec![select_ext], ExtSettings::builder() .log_level(&self.log_level) .server_info(server_info) @@ -134,6 +136,18 @@ fn load_console() -> &'static [u8] { b"" } +#[cfg(feature = "select-ext")] +fn load_select_ext() -> &'static [u8] { + info!("Load Reduct Select Extension"); + include_bytes!(concat!(env!("OUT_DIR"), "/select-ext.zip")) +} + +#[cfg(not(feature = "select-ext"))] +fn load_select_ext() -> &'static [u8] { + info!("Reduct Select Extension is disabled"); + b"" +} + impl Display for Cfg { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.env.message()) diff --git a/reductstore/src/cfg/provision/bucket.rs b/reductstore/src/cfg/provision/bucket.rs index 60ad09fe7..050ff21ba 100644 --- a/reductstore/src/cfg/provision/bucket.rs +++ b/reductstore/src/cfg/provision/bucket.rs @@ -192,7 +192,7 @@ mod tests { mock_getter .expect_get() .with(eq("RS_DATA_PATH")) - .return_const(Ok(tmp.into_path().to_str().unwrap().to_string())); + .return_const(Ok(tmp.keep().to_str().unwrap().to_string())); mock_getter.expect_all().returning(|| { let mut map = BTreeMap::new(); map.insert("RS_BUCKET_1_NAME".to_string(), "bucket1".to_string()); diff --git a/reductstore/src/cfg/provision/replication.rs b/reductstore/src/cfg/provision/replication.rs index 7fd25a390..0982e9ff3 100644 --- a/reductstore/src/cfg/provision/replication.rs +++ b/reductstore/src/cfg/provision/replication.rs @@ -400,7 +400,7 @@ mod tests { #[fixture] fn path() -> PathBuf { let tmp = tempfile::tempdir().unwrap(); - tmp.into_path() + tmp.keep() } #[fixture] diff --git a/reductstore/src/cfg/provision/token.rs b/reductstore/src/cfg/provision/token.rs index 823c7c1df..bfbc7aa99 100644 --- a/reductstore/src/cfg/provision/token.rs +++ b/reductstore/src/cfg/provision/token.rs @@ -200,7 +200,7 @@ mod tests { mock_getter .expect_get() .with(eq("RS_DATA_PATH")) - .return_const(Ok(tmp.into_path().to_str().unwrap().to_string())); + .return_const(Ok(tmp.keep().to_str().unwrap().to_string())); mock_getter .expect_get() .with(eq("RS_API_TOKEN")) diff --git a/reductstore/src/core/file_cache.rs b/reductstore/src/core/file_cache.rs index a2c3d49bb..f15ad29f6 100644 --- a/reductstore/src/core/file_cache.rs +++ b/reductstore/src/core/file_cache.rs @@ -390,6 +390,6 @@ mod tests { #[fixture] fn tmp_dir() -> PathBuf { - tempfile::tempdir().unwrap().into_path() + tempfile::tempdir().unwrap().keep() } } diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index d212649fb..75f631631 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -1,6 +1,7 @@ // Copyright 2025 ReductSoftware UG // Licensed under the Business Source License 1.1 +use crate::asset::asset_manager::ManageStaticAsset; use crate::ext::filter::ExtWhenFilter; use crate::storage::query::base::QueryOptions; use crate::storage::query::condition::{EvaluationStage, Parser}; @@ -58,49 +59,55 @@ struct ExtRepository { #[allow(dead_code)] ext_wrappers: Vec>, // we need to keep the wrappers alive + + #[allow(dead_code)] + embedded_extensions: Vec>, // we need to keep them from being cleaned up } impl ExtRepository { - pub(crate) fn try_load( - path: &PathBuf, + fn try_load( + paths: Vec, + embedded_extensions: Vec>, settings: ExtSettings, ) -> Result { let mut extension_map = IoExtMap::new(); let query_map = AsyncRwLock::new(HashMap::new()); + let mut ext_wrappers = Vec::new(); - if !path.exists() { - return Err(internal_server_error!( - "Extension directory {:?} does not exist", - path - )); - } + for path in paths { + if !path.exists() { + return Err(internal_server_error!( + "Extension directory {:?} does not exist", + path + )); + } - let mut ext_wrappers = Vec::new(); - for entry in path.read_dir()? { - let path = entry?.path(); - if path.is_file() - && path - .extension() - .map_or(false, |ext| ext == "so" || ext == "dll" || ext == "dylib") - { - let ext_wrapper = unsafe { - match Container::::load(path.clone()) { - Ok(wrapper) => wrapper, - Err(e) => { - error!("Failed to load extension '{:?}': {:?}", path, e); - continue; + for entry in path.read_dir()? { + let path = entry?.path(); + if path.is_file() + && path + .extension() + .map_or(false, |ext| ext == "so" || ext == "dll" || ext == "dylib") + { + let ext_wrapper = unsafe { + match Container::::load(path.clone()) { + Ok(wrapper) => wrapper, + Err(e) => { + error!("Failed to load extension '{:?}': {:?}", path, e); + continue; + } } - } - }; + }; - let ext = unsafe { Box::from_raw(ext_wrapper.get_ext(settings.clone())) }; + let ext = unsafe { Box::from_raw(ext_wrapper.get_ext(settings.clone())) }; - info!("Load extension: {:?}", ext.info()); + info!("Load extension: {:?}", ext.info()); - let name = ext.info().name().to_string(); - extension_map.insert(name, Arc::new(AsyncRwLock::new(ext))); - ext_wrappers.push(ext_wrapper); + let name = ext.info().name().to_string(); + extension_map.insert(name, Arc::new(AsyncRwLock::new(ext))); + ext_wrappers.push(ext_wrapper); + } } } @@ -108,6 +115,7 @@ impl ExtRepository { extension_map, query_map, ext_wrappers, + embedded_extensions, }) } @@ -282,11 +290,28 @@ impl ManageExtensions for ExtRepository { } pub fn create_ext_repository( - path: Option, + external_path: Option, + embedded_extensions: Vec>, settings: ExtSettings, ) -> Result { - if let Some(path) = path { - Ok(Box::new(ExtRepository::try_load(&path, settings)?)) + if external_path.is_some() || !embedded_extensions.is_empty() { + let mut paths = if let Some(path) = external_path { + vec![path] + } else { + Vec::new() + }; + + for embedded in &embedded_extensions { + if let Ok(path) = embedded.absolut_path("") { + paths.push(path); + } + } + + Ok(Box::new(ExtRepository::try_load( + paths, + embedded_extensions, + settings, + )?)) } else { // Dummy extension repository if struct NoExtRepository; @@ -364,17 +389,17 @@ pub(super) mod tests { #[log_test(rstest)] fn test_failed_load(ext_settings: ExtSettings) { - let path = tempdir().unwrap().into_path(); + let path = tempdir().unwrap().keep(); fs::create_dir_all(&path).unwrap(); fs::write(&path.join("libtest.so"), b"test").unwrap(); - let ext_repo = ExtRepository::try_load(&path, ext_settings).unwrap(); + let ext_repo = ExtRepository::try_load(vec![path], vec![], ext_settings).unwrap(); assert_eq!(ext_repo.extension_map.len(), 0); } #[log_test(rstest)] fn test_failed_open_dir(ext_settings: ExtSettings) { let path = PathBuf::from("non_existing_dir"); - let ext_repo = ExtRepository::try_load(&path, ext_settings); + let ext_repo = ExtRepository::try_load(vec![path], vec![], ext_settings); assert_eq!( ext_repo.err().unwrap(), internal_server_error!("Extension directory \"non_existing_dir\" does not exist") @@ -408,7 +433,7 @@ pub(super) mod tests { panic!("Unsupported platform") }; - let ext_path = PathBuf::from(tempdir().unwrap().into_path()).join("ext"); + let ext_path = PathBuf::from(tempdir().unwrap().keep()).join("ext"); fs::create_dir_all(ext_path.clone()).unwrap(); let link = format!( @@ -429,7 +454,8 @@ pub(super) mod tests { fs::write(ext_path.join(file_name), resp.bytes().unwrap()) .expect("Failed to write extension"); - ExtRepository::try_load(&ext_path.to_path_buf(), ext_settings).unwrap() + let empty_ext_path = tempdir().unwrap().keep(); + ExtRepository::try_load(vec![ext_path, empty_ext_path], vec![], ext_settings).unwrap() } } mod register_query { @@ -827,7 +853,7 @@ pub(super) mod tests { .server_info(ServerInfo::default()) .build(); let mut ext_repo = - ExtRepository::try_load(&tempdir().unwrap().into_path(), ext_settings).unwrap(); + ExtRepository::try_load(vec![tempdir().unwrap().keep()], vec![], ext_settings).unwrap(); ext_repo.extension_map.insert( name.to_string(), Arc::new(AsyncRwLock::new(Box::new(mock_ext))), diff --git a/reductstore/src/license.rs b/reductstore/src/license.rs index accf146ed..905307d28 100644 --- a/reductstore/src/license.rs +++ b/reductstore/src/license.rs @@ -97,7 +97,7 @@ mod tests { } "#; - let license_path = tempdir().unwrap().into_path().join("license.jwt"); + let license_path = tempdir().unwrap().keep().join("license.jwt"); fs::write(&license_path, license_file).unwrap(); license_path } @@ -111,7 +111,7 @@ mod tests { } "#; - let license_path = tempdir().unwrap().into_path().join("license.jwt"); + let license_path = tempdir().unwrap().keep().join("license.jwt"); fs::write(&license_path, license_file).unwrap(); license_path } diff --git a/reductstore/src/main.rs b/reductstore/src/main.rs index e202c0b4c..6359f9ae8 100644 --- a/reductstore/src/main.rs +++ b/reductstore/src/main.rs @@ -232,12 +232,12 @@ mod tests { #[tokio::test] async fn test_shutdown() { let handle = Handle::new(); - let storage = Arc::new(Storage::load(tempdir().unwrap().into_path(), None)); + let storage = Arc::new(Storage::load(tempdir().unwrap().keep(), None)); shutdown_app(handle.clone(), storage.clone()); } async fn set_env_and_run(cfg: HashMap) -> JoinHandle<()> { - let data_path = tempdir().unwrap().into_path(); + let data_path = tempdir().unwrap().keep(); env::set_var("RS_DATA_PATH", data_path.to_str().unwrap()); env::set_var("RS_CERT_PATH", ""); diff --git a/reductstore/src/replication/replication_repository.rs b/reductstore/src/replication/replication_repository.rs index e30accdb2..69635bd8b 100644 --- a/reductstore/src/replication/replication_repository.rs +++ b/reductstore/src/replication/replication_repository.rs @@ -734,7 +734,7 @@ mod tests { #[fixture] fn storage() -> Arc { let tmp_dir = tempfile::tempdir().unwrap(); - let storage = Storage::load(tmp_dir.into_path(), None); + let storage = Storage::load(tmp_dir.keep(), None); let bucket = storage .create_bucket("bucket-1", BucketSettings::default()) .unwrap() diff --git a/reductstore/src/replication/replication_sender.rs b/reductstore/src/replication/replication_sender.rs index 76bf41d9e..c2fcaadb1 100644 --- a/reductstore/src/replication/replication_sender.rs +++ b/reductstore/src/replication/replication_sender.rs @@ -626,7 +626,7 @@ mod tests { remote_bucket: MockRmBucket, settings: ReplicationSettings, ) -> ReplicationSender { - let tmp_dir = tempfile::tempdir().unwrap().into_path(); + let tmp_dir = tempfile::tempdir().unwrap().keep(); let storage = Arc::new(Storage::load(tmp_dir, None)); diff --git a/reductstore/src/replication/replication_task.rs b/reductstore/src/replication/replication_task.rs index 8fc9a1f04..d4306cf20 100644 --- a/reductstore/src/replication/replication_task.rs +++ b/reductstore/src/replication/replication_task.rs @@ -555,7 +555,7 @@ mod tests { remote_bucket: MockRmBucket, settings: ReplicationSettings, ) -> ReplicationTask { - let tmp_dir = tempfile::tempdir().unwrap().into_path(); + let tmp_dir = tempfile::tempdir().unwrap().keep(); let storage = Arc::new(Storage::load(tmp_dir, None)); diff --git a/reductstore/src/replication/transaction_log.rs b/reductstore/src/replication/transaction_log.rs index 2940d8879..be757eddb 100644 --- a/reductstore/src/replication/transaction_log.rs +++ b/reductstore/src/replication/transaction_log.rs @@ -445,7 +445,7 @@ mod tests { #[fixture] fn path() -> PathBuf { - let path = tempdir().unwrap().into_path().join("transaction_log"); + let path = tempdir().unwrap().keep().join("transaction_log"); path } } diff --git a/reductstore/src/storage/block_manager.rs b/reductstore/src/storage/block_manager.rs index 92dbdae4b..d4cde46d2 100644 --- a/reductstore/src/storage/block_manager.rs +++ b/reductstore/src/storage/block_manager.rs @@ -1073,7 +1073,7 @@ mod tests { #[fixture] fn block_manager(block_id: u64) -> BlockManager { - let path = tempdir().unwrap().into_path().join("bucket").join("entry"); + let path = tempdir().unwrap().keep().join("bucket").join("entry"); let mut bm = BlockManager::new(path.clone(), BlockIndex::new(path.join(BLOCK_INDEX_FILE))); let block_ref = bm.start_new_block(block_id, 1024).unwrap().clone(); diff --git a/reductstore/src/storage/block_manager/block_index.rs b/reductstore/src/storage/block_manager/block_index.rs index 031b44d14..588694fb3 100644 --- a/reductstore/src/storage/block_manager/block_index.rs +++ b/reductstore/src/storage/block_manager/block_index.rs @@ -293,7 +293,7 @@ mod tests { #[rstest] fn test_ok() { - let path = tempdir().unwrap().into_path().join(BLOCK_INDEX_FILE); + let path = tempdir().unwrap().keep().join(BLOCK_INDEX_FILE); let block_index_proto = BlockIndexProto { blocks: vec![BlockEntry { @@ -327,7 +327,7 @@ mod tests { #[rstest] fn test_index_file_corrupted() { - let path = tempdir().unwrap().into_path().join(BLOCK_INDEX_FILE); + let path = tempdir().unwrap().keep().join(BLOCK_INDEX_FILE); let block_index_proto = BlockIndexProto { blocks: vec![BlockEntry { @@ -351,7 +351,7 @@ mod tests { #[rstest] fn test_decode_err() { - let path = tempdir().unwrap().into_path().join(BLOCK_INDEX_FILE); + let path = tempdir().unwrap().keep().join(BLOCK_INDEX_FILE); fs::write(&path, vec![0, 1, 2, 3]).unwrap(); let block_index = BlockIndex::try_load(path.clone()).err().unwrap(); @@ -364,7 +364,7 @@ mod tests { #[rstest] fn test_ok() { - let path = tempdir().unwrap().into_path().join(BLOCK_INDEX_FILE); + let path = tempdir().unwrap().keep().join(BLOCK_INDEX_FILE); let mut block_index = BlockIndex::new(path.clone()); block_index.insert_or_update(BlockEntry { diff --git a/reductstore/src/storage/block_manager/wal.rs b/reductstore/src/storage/block_manager/wal.rs index df419761a..1992669ba 100644 --- a/reductstore/src/storage/block_manager/wal.rs +++ b/reductstore/src/storage/block_manager/wal.rs @@ -377,7 +377,7 @@ mod tests { #[fixture] fn wal() -> WalImpl { - let path = tempfile::tempdir().unwrap().into_path(); + let path = tempfile::tempdir().unwrap().keep(); std::fs::create_dir_all(path.join("wal")).unwrap(); WalImpl::new(path.join("wal")) } diff --git a/reductstore/src/storage/bucket.rs b/reductstore/src/storage/bucket.rs index 20ac581f5..0b80bff0b 100644 --- a/reductstore/src/storage/bucket.rs +++ b/reductstore/src/storage/bucket.rs @@ -581,7 +581,7 @@ mod tests { #[fixture] pub fn path() -> PathBuf { - tempdir().unwrap().into_path() + tempdir().unwrap().keep() } #[fixture] diff --git a/reductstore/src/storage/entry.rs b/reductstore/src/storage/entry.rs index 6efb5e166..0e8a1239a 100644 --- a/reductstore/src/storage/entry.rs +++ b/reductstore/src/storage/entry.rs @@ -620,7 +620,7 @@ mod tests { #[fixture] pub(super) fn path() -> PathBuf { - tempfile::tempdir().unwrap().into_path().join("bucket") + tempfile::tempdir().unwrap().keep().join("bucket") } pub fn write_record(entry: &mut Entry, time: u64, data: Vec) { diff --git a/reductstore/src/storage/entry/io/record_reader.rs b/reductstore/src/storage/entry/io/record_reader.rs index e43570706..8733b1f9a 100644 --- a/reductstore/src/storage/entry/io/record_reader.rs +++ b/reductstore/src/storage/entry/io/record_reader.rs @@ -368,7 +368,7 @@ mod tests { } #[fixture] fn file_to_read(content_size: usize) -> PathBuf { - let tmp_file = tempdir().unwrap().into_path().join("test_file"); + let tmp_file = tempdir().unwrap().keep().join("test_file"); std::fs::write(&tmp_file, vec![0; content_size]).unwrap(); tmp_file diff --git a/reductstore/src/storage/entry/io/record_writer.rs b/reductstore/src/storage/entry/io/record_writer.rs index 4073b162a..4dd00c5b6 100644 --- a/reductstore/src/storage/entry/io/record_writer.rs +++ b/reductstore/src/storage/entry/io/record_writer.rs @@ -421,7 +421,7 @@ mod tests { #[fixture] fn path() -> PathBuf { - tempdir().unwrap().into_path().join("bucket").join("entry") + tempdir().unwrap().keep().join("bucket").join("entry") } #[fixture] diff --git a/reductstore/src/storage/query.rs b/reductstore/src/storage/query.rs index 306fb2774..8b4c22010 100644 --- a/reductstore/src/storage/query.rs +++ b/reductstore/src/storage/query.rs @@ -315,7 +315,7 @@ mod tests { fn block_manager() -> Arc> { let path = tempfile::tempdir() .unwrap() - .into_path() + .keep() .join("bucket") .join("entry"); diff --git a/reductstore/src/storage/query/base.rs b/reductstore/src/storage/query/base.rs index ea682d22c..48f666076 100644 --- a/reductstore/src/storage/query/base.rs +++ b/reductstore/src/storage/query/base.rs @@ -117,7 +117,7 @@ pub(crate) mod tests { // Two blocks // the first block has two records: 0, 5 // the second block has a record: 1000 - let dir = tempdir().unwrap().into_path().join("bucket").join("entry"); + let dir = tempdir().unwrap().keep().join("bucket").join("entry"); let mut block_manager = BlockManager::new(dir.clone(), BlockIndex::new(dir.join("index"))); let block_ref = block_manager.start_new_block(0, 10).unwrap(); diff --git a/reductstore/src/storage/storage.rs b/reductstore/src/storage/storage.rs index 0845fea77..bb792ae58 100644 --- a/reductstore/src/storage/storage.rs +++ b/reductstore/src/storage/storage.rs @@ -654,7 +654,7 @@ mod tests { #[fixture] fn path() -> PathBuf { - tempdir().unwrap().into_path() + tempdir().unwrap().keep() } #[fixture] From 749bf74196aae5764a0284bc3d0d6080964f72d0 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 15 May 2025 14:49:49 +0200 Subject: [PATCH 08/93] fix integration tests --- integration_tests/api/ext_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/api/ext_test.py b/integration_tests/api/ext_test.py index 141e39d98..371d40977 100644 --- a/integration_tests/api/ext_test.py +++ b/integration_tests/api/ext_test.py @@ -1,6 +1,6 @@ import json -from integration_tests.api.conftest import requires_env +from .conftest import requires_env @requires_env("LICENSE_PATH") From 8a55c3e53208bc0b722047f15636fcb044cd3a8f Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 15 May 2025 16:38:26 +0200 Subject: [PATCH 09/93] update license key test --- integration_tests/api/server_api_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/api/server_api_test.py b/integration_tests/api/server_api_test.py index dc712a878..b2c494f3c 100644 --- a/integration_tests/api/server_api_test.py +++ b/integration_tests/api/server_api_test.py @@ -35,12 +35,12 @@ def test__get_lic_info(base_url, session): data = json.loads(resp.content) assert data["license"] == { "device_number": 1, - "disk_quota": 0, - "expiry_date": "2035-01-01T00:00:00Z", - "fingerprint": "df92c95a7c9b56c2af99b290c39d8471c3e6cbf9dc33dc9bdb4116b98d465cc9", - "invoice": "xxxxxx", - "licensee": "ReductStore,LLC", - "plan": "UNLIMITED", + "disk_quota": 1, + "expiry_date": "2026-05-15T13:35:43.696974Z", + "fingerprint": "21e2608b7d47f7fba623d714c3e14b73cd1fe3578f4010ef26bcbedfc42a4c92", + "invoice": "---", + "licensee": "ReductSoftware", + "plan": "STANDARD", } From b451cb7f557ceff86072687bf87a0fd903c7e1e4 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 15 May 2025 17:57:18 +0200 Subject: [PATCH 10/93] enable reduct select in docker --- buildx.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildx.Dockerfile b/buildx.Dockerfile index 373a1d3a8..e6d126774 100644 --- a/buildx.Dockerfile +++ b/buildx.Dockerfile @@ -35,8 +35,8 @@ COPY Cargo.lock Cargo.lock RUN cargo install --force --locked bindgen-cli -RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --target ${CARGO_TARGET} --package reductstore -RUN cargo install reduct-cli --target ${CARGO_TARGET} --root /src/target/${CARGO_TARGET}/release --all-features +RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --target ${CARGO_TARGET} --package reductstore --all-features +RUN cargo install reduct-cli --target ${CARGO_TARGET} --root /src/target/${CARGO_TARGET}/release RUN mkdir /data From b504d5cf78abcaf64fc91eb33545089f82ea51d7 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 16 May 2025 08:07:20 +0200 Subject: [PATCH 11/93] fix docker build --- .github/workflows/ci.yml | 4 ++-- Dockerfile | 5 +++-- buildx.Dockerfile | 5 ++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4db9720a1..f3f1df288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: tags: ${{github.repository}}:latest outputs: type=docker,dest=/tmp/image.tar build-args: | - ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} + ARTIFACT_SAS_URL="${{ secrets.ARTIFACT_SAS_URL }}" RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} - name: Upload artifact @@ -732,7 +732,7 @@ jobs: GIT_COMMIT=${{env.GITHUB_SHA}} CARGO_TARGET=${{matrix.cargo_target}} GCC_COMPILER=${{matrix.gcc_compiler}} - ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} + ARTIFACT_SAS_URL="${{ secrets.ARTIFACT_SAS_URL }}" RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true diff --git a/Dockerfile b/Dockerfile index 15f78dc10..07e1a68ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM ubuntu:22.04 AS builder -ARG GIT_COMMIT=unspecified -ARG ARTIFACT_SAS_URL + ARG RUST_VERSION RUN apt-get update && apt-get install -y \ @@ -25,6 +24,8 @@ COPY .cargo .cargo COPY Cargo.toml Cargo.toml COPY Cargo.lock Cargo.lock +ARG GIT_COMMIT=unspecified +ARG ARTIFACT_SAS_URL RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --all-features FROM ubuntu:22.04 diff --git a/buildx.Dockerfile b/buildx.Dockerfile index e6d126774..56461f908 100644 --- a/buildx.Dockerfile +++ b/buildx.Dockerfile @@ -6,8 +6,6 @@ ARG TARGETPLATFORM ARG BUILDPLATFORM ARG CARGO_TARGET ARG GCC_COMPILER=gcc-11 -ARG GIT_COMMIT=unspecified -ARG ARTIFACT_SAS_URL ARG RUST_VERSION RUN apt-get update && apt-get install -y \ @@ -33,7 +31,8 @@ COPY .cargo /root/.cargo COPY Cargo.toml Cargo.toml COPY Cargo.lock Cargo.lock - +ARG GIT_COMMIT=unspecified +ARG ARTIFACT_SAS_URL RUN cargo install --force --locked bindgen-cli RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --target ${CARGO_TARGET} --package reductstore --all-features RUN cargo install reduct-cli --target ${CARGO_TARGET} --root /src/target/${CARGO_TARGET}/release From b398b8137c2fb9c85dc08beb6ecdd6c7e6c0d81b Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 16 May 2025 08:17:07 +0200 Subject: [PATCH 12/93] use only one dockerfile --- .github/workflows/ci.yml | 5 +++-- Dockerfile | 39 --------------------------------------- 2 files changed, 3 insertions(+), 41 deletions(-) delete mode 100644 Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3f1df288..a1773b553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,10 +42,11 @@ jobs: uses: docker/build-push-action@v4 with: context: . + file: buildx.Dockerfile tags: ${{github.repository}}:latest outputs: type=docker,dest=/tmp/image.tar build-args: | - ARTIFACT_SAS_URL="${{ secrets.ARTIFACT_SAS_URL }}" + ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} - name: Upload artifact @@ -732,7 +733,7 @@ jobs: GIT_COMMIT=${{env.GITHUB_SHA}} CARGO_TARGET=${{matrix.cargo_target}} GCC_COMPILER=${{matrix.gcc_compiler}} - ARTIFACT_SAS_URL="${{ secrets.ARTIFACT_SAS_URL }}" + ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 07e1a68ad..000000000 --- a/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM ubuntu:22.04 AS builder - - -ARG RUST_VERSION - -RUN apt-get update && apt-get install -y \ - cmake \ - build-essential \ - curl \ - protobuf-compiler - -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - -# Add .cargo/bin to PATH -ENV PATH="/root/.cargo/bin:${PATH}" -RUN rustup default ${RUST_VERSION} - -WORKDIR /src - -COPY reductstore reductstore -COPY reduct_base reduct_base -COPY reduct_macros reduct_macros -COPY .cargo .cargo -COPY Cargo.toml Cargo.toml -COPY Cargo.lock Cargo.lock - -ARG GIT_COMMIT=unspecified -ARG ARTIFACT_SAS_URL -RUN GIT_COMMIT=${GIT_COMMIT} ARTIFACT_SAS_URL=${ARTIFACT_SAS_URL} cargo build --release --all-features - -FROM ubuntu:22.04 - -COPY --from=builder /src/target/release/reductstore /usr/local/bin/reductstore - -EXPOSE 8383 - -RUN mkdir /data - -CMD ["reductstore"] From 7ae2531cc54848bfc9806d0ae4b1cf7bf5e6cdd8 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 16 May 2025 09:03:26 +0200 Subject: [PATCH 13/93] use select-ext v0.1.1 with linux arm32 --- reductstore/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reductstore/build.rs b/reductstore/build.rs index a366d7c67..1085da51f 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -30,7 +30,7 @@ fn main() -> Result<(), Box> { download_web_console("v1.10.0"); #[cfg(feature = "select-ext")] - download_ext("select-ext", "v0.1.0"); + download_ext("select-ext", "v0.1.1"); // get build time and commit let build_time = chrono::DateTime::::from(SystemTime::now()) From 3fcc2449e526201653510ef4f9354e05e77fba1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 May 2025 08:37:24 +0200 Subject: [PATCH 14/93] Bump zip from 2.6.1 to 3.0.0 (#824) Bumps [zip](https://github.com/zip-rs/zip2) from 2.6.1 to 3.0.0. - [Release notes](https://github.com/zip-rs/zip2/releases) - [Changelog](https://github.com/zip-rs/zip2/blob/master/CHANGELOG.md) - [Commits](https://github.com/zip-rs/zip2/compare/v2.6.1...v3.0.0) --- updated-dependencies: - dependency-name: zip dependency-version: 3.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 25 ++++++++++++++++++++----- reductstore/Cargo.toml | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06c7cf8ce..9b0a656e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -725,11 +725,12 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -1407,6 +1408,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -3490,16 +3500,15 @@ dependencies = [ [[package]] name = "zip" -version = "2.6.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" dependencies = [ "aes", "arbitrary", "bzip2", "constant_time_eq", "crc32fast", - "crossbeam-utils", "deflate64", "flate2", "getrandom 0.3.2", @@ -3516,6 +3525,12 @@ dependencies = [ "zstd", ] +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zopfli" version = "0.8.1" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 0af9ab6b8..4eb64d72b 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -31,7 +31,7 @@ reduct-base = { path = "../reduct_base", version = "1.15.0", features = ["ext"] reduct-macros = { path = "../reduct_macros", version = "1.15.0" } chrono = { version = "0.4.41", features = ["serde"] } -zip = "2.6.1" +zip = "3.0.0" tempfile = "3.20.0" hex = "0.4.3" prost-wkt-types = "0.6.1" From 2b1b12900cb50f8f15fe42f369760d33a6e52099 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 23 May 2025 17:56:16 +0200 Subject: [PATCH 15/93] Refactor Extension API for multi-line CSV processing (#823) * refactor ext api for batching * filter sync+send * refactor record meta * refactor api * remove register query * remove query id * use reduct-select v0.2.0 * update CHANGELOG --- CHANGELOG.md | 1 + Cargo.lock | 1 + integration_tests/api/ext_test.py | 2 +- reduct_base/Cargo.toml | 2 +- reduct_base/src/ext.rs | 92 +- reduct_base/src/ext/process_status.rs | 50 - reduct_base/src/io.rs | 256 ++++- reductstore/build.rs | 2 +- reductstore/src/api/entry/read_batched.rs | 114 +-- reductstore/src/api/entry/read_query.rs | 2 +- reductstore/src/api/entry/read_query_post.rs | 2 +- reductstore/src/api/entry/read_single.rs | 22 +- reductstore/src/api/entry/update_batched.rs | 8 +- reductstore/src/api/entry/update_single.rs | 8 +- reductstore/src/api/entry/write_batched.rs | 24 +- reductstore/src/api/entry/write_single.rs | 4 +- reductstore/src/ext/ext_repository.rs | 915 ++++++++---------- reductstore/src/ext/ext_repository/create.rs | 69 ++ reductstore/src/ext/ext_repository/load.rs | 172 ++++ reductstore/src/ext/filter.rs | 156 +-- reductstore/src/replication.rs | 7 + .../remote_bucket/client_wrapper.rs | 17 +- .../remote_bucket/states/bucket_available.rs | 6 +- .../src/replication/replication_sender.rs | 2 +- .../src/replication/transaction_filter.rs | 25 +- reductstore/src/storage/entry.rs | 12 +- reductstore/src/storage/entry/entry_loader.rs | 14 +- .../src/storage/entry/io/record_reader.rs | 80 +- reductstore/src/storage/entry/read_record.rs | 4 +- .../src/storage/entry/remove_records.rs | 4 +- .../src/storage/entry/update_labels.rs | 8 +- reductstore/src/storage/proto.rs | 19 + reductstore/src/storage/query.rs | 12 +- reductstore/src/storage/query/continuous.rs | 9 +- reductstore/src/storage/query/filters.rs | 49 +- .../src/storage/query/filters/each_n.rs | 16 +- .../src/storage/query/filters/each_s.rs | 37 +- .../src/storage/query/filters/exclude.rs | 82 +- .../src/storage/query/filters/include.rs | 83 +- .../src/storage/query/filters/record_state.rs | 22 +- .../src/storage/query/filters/time_range.rs | 41 +- reductstore/src/storage/query/filters/when.rs | 26 +- reductstore/src/storage/query/historical.rs | 73 +- reductstore/src/storage/query/limited.rs | 6 +- 44 files changed, 1298 insertions(+), 1258 deletions(-) delete mode 100644 reduct_base/src/ext/process_status.rs create mode 100644 reductstore/src/ext/ext_repository/create.rs create mode 100644 reductstore/src/ext/ext_repository/load.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f9e81086..a091ebb9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) +- Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) ## [1.15.2] - 2025-05-21 diff --git a/Cargo.lock b/Cargo.lock index 9b0a656e8..e50d8faac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2016,6 +2016,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "futures", "http", "int-enum", "log", diff --git a/integration_tests/api/ext_test.py b/integration_tests/api/ext_test.py index 371d40977..41384e74c 100644 --- a/integration_tests/api/ext_test.py +++ b/integration_tests/api/ext_test.py @@ -25,4 +25,4 @@ def test__select_ext(base_url, bucket, session): resp = session.get(f"{base_url}/b/{bucket}/entry/batch?q={query_id}") assert resp.status_code == 200 - assert resp.headers["x-reduct-time-1"] == "2,application/octet-stream,@a=1" + assert resp.headers["x-reduct-time-1"] == "2,text/csv,@a=1" diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 3c9d10c9a..27187d768 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -37,7 +37,7 @@ async-trait = { version = "0.1.87" , optional = true } tokio = { version = "1.45.0", optional = true, features = ["default", "rt", "time"] } log = "0.4.0" thread-id = "5.0.0" - +futures = "0.3.31" [dev-dependencies] rstest = "0.25.0" diff --git a/reduct_base/src/ext.rs b/reduct_base/src/ext.rs index 0d465eb0e..880bbe2be 100644 --- a/reduct_base/src/ext.rs +++ b/reduct_base/src/ext.rs @@ -5,22 +5,61 @@ mod ext_info; mod ext_settings; -mod process_status; use crate::error::ReductError; use crate::io::ReadRecord; use crate::msg::entry_api::QueryEntry; use async_trait::async_trait; - pub use ext_info::{IoExtensionInfo, IoExtensionInfoBuilder}; - -pub use process_status::ProcessStatus; +use futures::stream::Stream; pub use ext_settings::{ExtSettings, ExtSettingsBuilder}; pub type BoxedReadRecord = Box; +pub type BoxedRecordStream = + Box> + Send + Sync>; pub const EXTENSION_API_VERSION: &str = "0.2"; +#[async_trait] +pub trait Commiter { + /// Commit record after processing and filtering. + /// + /// This method is called after processing and filtering the record and + /// can be used to rebatch records when they represent entries of some data format like CVS lines, or JSON objects. + /// An extension can concatenate multiple records into one or split one record into multiple records depending on the query. + async fn commit_record( + &mut self, + record: BoxedReadRecord, + ) -> Option>; + + /// Flush the rest of the records. + async fn flush(&mut self) -> Option>; +} + +#[async_trait] +pub trait Processor { + /// Processes a record in the extension. + /// + /// This method is called for each record that is fetched from the storage engine. + /// + /// # Arguments + /// + /// * `query_id` - The ID of the query. + /// * `record` - The record to process. + /// + /// # Returns + //// + /// A stream of records that are processed by the extension. If the input represents data that has multiple entries, + /// the extension can return a stream of records that are processed by the extension for each entry. + async fn process_record( + &mut self, + record: BoxedReadRecord, + ) -> Result; +} + +pub type BoxedCommiter = Box; +pub type BoxedProcessor = Box; + /// The trait for the IO extension. /// /// This trait is used to register queries and process records in a pipeline of extensions. @@ -41,49 +80,18 @@ pub trait IoExtension { /// /// # Arguments /// - /// * `query_id` - The ID of the query. /// * `bucket_name` - The name of the bucket. /// * `entry_name` - The name of the entry. /// * `query` - The query options - fn register_query( - &mut self, - query_id: u64, - bucket_name: &str, - entry_name: &str, - query: &QueryEntry, - ) -> Result<(), ReductError>; - - /// Unregisters a query in the extension. - /// - /// This method is called after fetching records from the storage engine. - /// - /// # Arguments - /// - /// * `query_id` - The ID of the query. - /// - /// # Returns - /// - /// The status of the unregistering of the query. - fn unregister_query(&mut self, query_id: u64) -> Result<(), ReductError>; - - /// Processes a record in the extension. - /// - /// This method is called for each record that is fetched from the storage engine. - /// - /// # Arguments - /// - /// * `query_id` - The ID of the query. - /// * `record` - The record to process. /// /// # Returns /// - /// The status of the processing of the record. - /// Ready status means that the record is ready to be processed by the next extension in the pipeline. - /// NotReady status means that the record is not ready to be processed by the next extension in the pipeline, but the pipeline should continue. - /// Stop status means that the pipeline should stop processing records. - async fn next_processed_record( + /// BoxedProcessor to process the data in the extension and return internal entries as temporary records. + /// BoxedCommiter to commit the records after processing and filtering into the final records. + fn query( &mut self, - query_id: u64, - record: BoxedReadRecord, - ) -> ProcessStatus; + bucket_name: &str, + entry_name: &str, + query: &QueryEntry, + ) -> Result<(BoxedProcessor, BoxedCommiter), ReductError>; } diff --git a/reduct_base/src/ext/process_status.rs b/reduct_base/src/ext/process_status.rs deleted file mode 100644 index b2e5bf3b9..000000000 --- a/reduct_base/src/ext/process_status.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2025 ReductSoftware UG -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use crate::error::ReductError; -use crate::ext::BoxedReadRecord; -use std::fmt::Debug; - -/// The status of the processing of a record. -/// -/// The three possible states allow to aggregate records on the extension side. -pub enum ProcessStatus { - Ready(Result), - NotReady, - Stop, -} - -impl Debug for ProcessStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ProcessStatus::Ready(_) => write!(f, "Ready(?)"), - ProcessStatus::NotReady => write!(f, "NotReady"), - ProcessStatus::Stop => write!(f, "Stop"), - } - } -} - -#[cfg(test)] - -mod tests { - use super::*; - - use crate::io::tests::MockRecord; - - use rstest::rstest; - - #[rstest] - fn test_debug() { - let record: BoxedReadRecord = Box::new(MockRecord {}); - let status = ProcessStatus::Ready(Ok(record)); - assert_eq!(format!("{:?}", status), "Ready(?)"); - - let status = ProcessStatus::NotReady; - assert_eq!(format!("{:?}", status), "NotReady"); - - let status = ProcessStatus::Stop; - assert_eq!(format!("{:?}", status), "Stop"); - } -} diff --git a/reduct_base/src/io.rs b/reduct_base/src/io.rs index 097ab15d9..c8218ec31 100644 --- a/reduct_base/src/io.rs +++ b/reduct_base/src/io.rs @@ -8,22 +8,156 @@ use tokio::runtime::Handle; pub type WriteChunk = Result, ReductError>; pub type ReadChunk = Option>; -pub trait RecordMeta { +#[derive(Debug, Clone, PartialEq)] +pub struct RecordMeta { + timestamp: u64, + state: i32, + labels: Labels, + content_type: String, + content_length: u64, + computed_labels: Labels, + last: bool, +} + +pub struct BuilderRecordMeta { + timestamp: u64, + state: i32, + labels: Labels, + content_type: String, + content_length: u64, + computed_labels: Labels, + last: bool, +} + +impl BuilderRecordMeta { + pub fn timestamp(mut self, timestamp: u64) -> Self { + self.timestamp = timestamp; + self + } + + pub fn state(mut self, state: i32) -> Self { + self.state = state; + self + } + + pub fn labels(mut self, labels: Labels) -> Self { + self.labels = labels; + self + } + + pub fn content_type(mut self, content_type: String) -> Self { + self.content_type = content_type; + self + } + + pub fn content_length(mut self, content_length: u64) -> Self { + self.content_length = content_length; + self + } + + pub fn computed_labels(mut self, computed_labels: Labels) -> Self { + self.computed_labels = computed_labels; + self + } + + pub fn last(mut self, last: bool) -> Self { + self.last = last; + self + } + + /// Builds a `RecordMeta` instance from the builder. + pub fn build(self) -> RecordMeta { + RecordMeta { + timestamp: self.timestamp, + state: self.state, + labels: self.labels, + content_type: self.content_type, + content_length: self.content_length, + computed_labels: self.computed_labels, + last: self.last, + } + } +} + +impl RecordMeta { + /// Creates a builder for a new `RecordMeta` instance. + pub fn builder() -> BuilderRecordMeta { + BuilderRecordMeta { + timestamp: 0, + state: 0, + labels: Labels::new(), + content_type: "application/octet-stream".to_string(), + content_length: 0, + computed_labels: Labels::new(), + last: false, + } + } + + /// Creates a builder from an existing `RecordMeta` instance. + pub fn builder_from(meta: RecordMeta) -> BuilderRecordMeta { + BuilderRecordMeta { + timestamp: meta.timestamp, + state: meta.state, + labels: meta.labels, + content_type: meta.content_type, + content_length: meta.content_length, + computed_labels: meta.computed_labels, + last: meta.last, + } + } + /// Returns the timestamp of the record as Unix time in microseconds. - fn timestamp(&self) -> u64; + pub fn timestamp(&self) -> u64 { + self.timestamp + } /// Returns the labels associated with the record. - fn labels(&self) -> &Labels; + pub fn labels(&self) -> &Labels { + &self.labels + } /// For filtering unfinished records. - fn state(&self) -> i32 { - 0 + pub fn state(&self) -> i32 { + self.state + } + + /// Returns true if this is the last record in the stream. + pub fn last(&self) -> bool { + self.last + } + + /// Returns computed labels associated with the record. + /// + /// Computed labels are labels that are added by query processing and are not part of the original record. + pub fn computed_labels(&self) -> &Labels { + &self.computed_labels + } + + /// Returns the labels associated with the record. + /// + /// Computed labels are labels that are added by query processing and are not part of the original record. + pub fn computed_labels_mut(&mut self) -> &mut Labels { + &mut self.computed_labels + } + + /// Returns the length of the record content in bytes. + pub fn content_length(&self) -> u64 { + self.content_length + } + + /// Returns the content type of the record as a MIME type. + pub fn content_type(&self) -> &str { + &self.content_type + } + + pub fn set_last(&mut self, last: bool) { + self.last = last; } } /// Represents a record in the storage engine that can be read as a stream of bytes. #[async_trait] -pub trait ReadRecord: RecordMeta { +pub trait ReadRecord { /// Reads a chunk of the record content. /// /// # Returns @@ -55,24 +189,8 @@ pub trait ReadRecord: RecordMeta { Handle::current().block_on(self.read()) } - /// Returns true if this is the last record in the stream. - fn last(&self) -> bool; - - /// Returns computed labels associated with the record. - /// - /// Computed labels are labels that are added by query processing and are not part of the original record. - fn computed_labels(&self) -> &Labels; - - /// Returns the labels associated with the record. - /// - /// Computed labels are labels that are added by query processing and are not part of the original record. - fn computed_labels_mut(&mut self) -> &mut Labels; - - /// Returns the length of the record content in bytes. - fn content_length(&self) -> u64; - - /// Returns the content type of the record as a MIME type. - fn content_type(&self) -> &str; + /// Returns meta information about the record. + fn meta(&self) -> &RecordMeta; } #[async_trait] @@ -94,6 +212,7 @@ pub trait WriteRecord { #[cfg(test)] pub(crate) mod tests { use super::*; + use rstest::rstest; use tokio::task::spawn_blocking; @@ -102,7 +221,7 @@ pub(crate) mod tests { #[tokio::test] async fn test_blocking_read() { let result = spawn_blocking(move || { - let mut record = MockRecord {}; + let mut record = MockRecord::new(); record.blocking_read() }); assert_eq!( @@ -114,7 +233,7 @@ pub(crate) mod tests { #[rstest] #[tokio::test] async fn test_default_read_timeout() { - let mut record = MockRecord {}; + let mut record = MockRecord::new(); let result = record.read_timeout(Duration::from_secs(1)).await; assert_eq!(result.unwrap(), Ok(Bytes::from_static(b"test"))); @@ -125,15 +244,68 @@ pub(crate) mod tests { ); } - pub struct MockRecord {} + mod meta { + use super::*; + + #[rstest] + fn test_builder() { + let meta = RecordMeta::builder() + .timestamp(1234567890) + .state(1) + .labels(Labels::new()) + .content_type("application/json".to_string()) + .content_length(1024) + .computed_labels(Labels::new()) + .last(true) + .build(); - impl RecordMeta for MockRecord { - fn timestamp(&self) -> u64 { - todo!() + assert_eq!(meta.timestamp(), 1234567890); + assert_eq!(meta.state(), 1); + assert_eq!(meta.content_type(), "application/json"); + assert_eq!(meta.content_length(), 1024); + assert_eq!(meta.last(), true); } - fn labels(&self) -> &Labels { - todo!() + #[rstest] + fn test_builder_from() { + let meta = RecordMeta::builder() + .timestamp(1234567890) + .state(1) + .labels(Labels::new()) + .content_type("application/json".to_string()) + .content_length(1024) + .computed_labels(Labels::new()) + .last(true) + .build(); + + let builder = RecordMeta::builder_from(meta.clone()); + let new_meta = builder.build(); + + assert_eq!(new_meta.timestamp(), 1234567890); + assert_eq!(new_meta.state(), 1); + assert_eq!(new_meta.content_type(), "application/json"); + assert_eq!(new_meta.content_length(), 1024); + assert_eq!(new_meta.last(), true); + } + } + + pub struct MockRecord { + metadata: RecordMeta, + } + + impl MockRecord { + pub fn new() -> Self { + Self { + metadata: RecordMeta::builder() + .timestamp(0) + .state(0) + .labels(Labels::new()) + .content_type("application/octet-stream".to_string()) + .content_length(0) + .computed_labels(Labels::new()) + .last(false) + .build(), + } } } @@ -144,24 +316,8 @@ pub(crate) mod tests { Some(Ok(Bytes::from("test"))) } - fn last(&self) -> bool { - todo!() - } - - fn computed_labels(&self) -> &Labels { - todo!() - } - - fn computed_labels_mut(&mut self) -> &mut Labels { - todo!() - } - - fn content_length(&self) -> u64 { - todo!() - } - - fn content_type(&self) -> &str { - todo!() + fn meta(&self) -> &RecordMeta { + &self.metadata } } } diff --git a/reductstore/build.rs b/reductstore/build.rs index 535bb2f7d..1c1cf2fe3 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -30,7 +30,7 @@ fn main() -> Result<(), Box> { download_web_console("v1.10.1"); #[cfg(feature = "select-ext")] - download_ext("select-ext", "v0.1.1"); + download_ext("select-ext", "v0.2.0"); // get build time and commit let build_time = chrono::DateTime::::from(SystemTime::now()) diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index 83cb129c4..e5085d8b5 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -20,7 +20,7 @@ use crate::storage::query::QueryRx; use futures_util::Future; use log::debug; use reduct_base::error::ReductError; -use reduct_base::ext::{BoxedReadRecord, ProcessStatus}; +use reduct_base::ext::BoxedReadRecord; use reduct_base::unprocessable_entity; use std::collections::HashMap; use std::pin::pin; @@ -75,10 +75,11 @@ pub(crate) async fn read_batched_records( } fn make_batch_header(reader: &BoxedReadRecord) -> (HeaderName, HeaderValue) { - let name = HeaderName::from_str(&format!("x-reduct-time-{}", reader.timestamp())).unwrap(); + let meta = reader.meta(); + let name = HeaderName::from_str(&format!("x-reduct-time-{}", meta.timestamp())).unwrap(); let mut meta_data = vec![ - reader.content_length().to_string(), - reader.content_type().to_string(), + meta.content_length().to_string(), + meta.content_type().to_string(), ]; let format_labels = |(k, v): (&String, &String)| { @@ -89,11 +90,10 @@ fn make_batch_header(reader: &BoxedReadRecord) -> (HeaderName, HeaderValue) { } }; - let mut labels: Vec = reader.labels().iter().map(format_labels).collect(); + let mut labels: Vec = meta.labels().iter().map(format_labels).collect(); labels.extend( - reader - .computed_labels() + meta.computed_labels() .iter() .map(|(k, v)| format_labels((&format!("@{}", k), v))), ); @@ -139,9 +139,8 @@ async fn fetch_and_response_batched_records( ) .await { - ProcessStatus::Ready(value) => value, - ProcessStatus::NotReady => continue, - ProcessStatus::Stop => break, + Some(value) => value, + None => continue, }; match reader { @@ -149,7 +148,7 @@ async fn fetch_and_response_batched_records( { let (name, value) = make_batch_header(&reader); header_size += name.as_str().len() + value.to_str().unwrap().len() + 2; - body_size += reader.content_length(); + body_size += reader.meta().content_length(); headers.insert(name, value); } readers.push(reader); @@ -210,18 +209,18 @@ async fn next_record_reader( query_path: &str, recv_timeout: Duration, ext_repository: &BoxedManageExtensions, -) -> ProcessStatus { +) -> Option> { // we need to wait for the first record if let Ok(result) = timeout( recv_timeout, - ext_repository.next_processed_record(query_id, rx), + ext_repository.fetch_and_process_record(query_id, rx), ) .await { result } else { debug!("Timeout while waiting for record from query {}", query_path); - ProcessStatus::Stop + None } } @@ -450,21 +449,19 @@ mod tests { let (_tx, rx) = tokio::sync::mpsc::channel(1); let rx = Arc::new(AsyncRwLock::new(rx)); assert!( - matches!( - timeout( - Duration::from_secs(1), - next_record_reader( - 1, - rx.clone(), - "", - Duration::from_millis(10), - &ext_repository - ) + timeout( + Duration::from_secs(1), + next_record_reader( + 1, + rx.clone(), + "", + Duration::from_millis(10), + &ext_repository ) - .await - .unwrap(), - ProcessStatus::Stop, - ), + ) + .await + .unwrap() + .is_none(), "should return None if the query is closed" ); } @@ -476,21 +473,19 @@ mod tests { let rx = Arc::new(AsyncRwLock::new(rx)); drop(tx); assert!( - matches!( - timeout( - Duration::from_secs(1), - next_record_reader( - 1, - rx.clone(), - "", - Duration::from_millis(0), - &ext_repository - ) + timeout( + Duration::from_secs(1), + next_record_reader( + 1, + rx.clone(), + "", + Duration::from_millis(0), + &ext_repository ) - .await - .unwrap(), - ProcessStatus::Stop - ), + ) + .await + .unwrap() + .is_none(), "should return None if the query is closed" ); } @@ -499,17 +494,14 @@ mod tests { #[rstest] fn test_batch_compute_labels() { let mut record = MockRecord::new(); - record.expect_timestamp().return_const(1000u64); - record.expect_content_length().return_const(100u64); - record - .expect_content_type() - .return_const("text/plain".to_string()); - record - .expect_labels() - .return_const(Labels::from_iter(vec![("a".to_string(), "b".to_string())])); - record - .expect_computed_labels() - .return_const(Labels::from_iter(vec![("x".to_string(), "y".to_string())])); + let meta = RecordMeta::builder() + .timestamp(1000u64) + .labels(Labels::from_iter(vec![("a".to_string(), "b".to_string())])) + .computed_labels(Labels::from_iter(vec![("x".to_string(), "y".to_string())])) + .content_length(100u64) + .content_type("text/plain".to_string()) + .build(); + record.expect_meta().return_const(meta); let record: BoxedReadRecord = Box::new(record); @@ -546,25 +538,11 @@ mod tests { mock! { Record {} - impl RecordMeta for Record { - fn timestamp(&self) -> u64; - - fn labels(&self) -> &Labels; - } - #[async_trait] impl ReadRecord for Record { async fn read(&mut self) -> Option>; - fn content_length(&self) -> u64; - - fn content_type(&self) -> &str; - - fn last(&self) -> bool; - - fn computed_labels(&self) -> &Labels; - - fn computed_labels_mut(&mut self) -> &mut Labels; + fn meta(&self) -> &RecordMeta; } } } diff --git a/reductstore/src/api/entry/read_query.rs b/reductstore/src/api/entry/read_query.rs index e565dd5fc..8f2148638 100644 --- a/reductstore/src/api/entry/read_query.rs +++ b/reductstore/src/api/entry/read_query.rs @@ -84,7 +84,7 @@ mod tests { .upgrade() .unwrap(); let mut rx = rx.write().await; - assert!(rx.recv().await.unwrap().unwrap().last()); + assert!(rx.recv().await.unwrap().unwrap().meta().last()); assert_eq!( rx.recv().await.unwrap().err().unwrap().status, diff --git a/reductstore/src/api/entry/read_query_post.rs b/reductstore/src/api/entry/read_query_post.rs index 425394f95..c493275f6 100644 --- a/reductstore/src/api/entry/read_query_post.rs +++ b/reductstore/src/api/entry/read_query_post.rs @@ -76,7 +76,7 @@ mod tests { .upgrade() .unwrap(); let mut rx = rx.write().await; - assert!(rx.recv().await.unwrap().unwrap().last()); + assert!(rx.recv().await.unwrap().unwrap().meta().last()); assert_eq!( rx.recv().await.unwrap().err().unwrap().status, ErrorCode::NoContent diff --git a/reductstore/src/api/entry/read_single.rs b/reductstore/src/api/entry/read_single.rs index 0328bb94a..00a8766d2 100644 --- a/reductstore/src/api/entry/read_single.rs +++ b/reductstore/src/api/entry/read_single.rs @@ -22,7 +22,7 @@ use crate::storage::query::QueryRx; use futures_util::Future; use hyper::http::HeaderValue; use reduct_base::bad_request; -use reduct_base::io::{ReadRecord, RecordMeta}; +use reduct_base::io::ReadRecord; use std::collections::HashMap; use std::i64; use std::pin::{pin, Pin}; @@ -70,7 +70,8 @@ async fn fetch_and_response_single_record( let make_headers = |record_reader: &RecordReader| { let mut headers = HeaderMap::new(); - for (k, v) in record_reader.labels() { + let meta = record_reader.meta(); + for (k, v) in meta.labels() { headers.insert( format!("x-reduct-label-{}", k) .parse::() @@ -81,20 +82,11 @@ async fn fetch_and_response_single_record( headers.insert( "content-type", - HeaderValue::from_str(record_reader.content_type()).unwrap(), - ); - headers.insert( - "content-length", - HeaderValue::from(record_reader.content_length()), - ); - headers.insert( - "x-reduct-time", - HeaderValue::from(record_reader.timestamp()), - ); - headers.insert( - "x-reduct-last", - HeaderValue::from(i64::from(record_reader.last())), + HeaderValue::from_str(meta.content_type()).unwrap(), ); + headers.insert("content-length", HeaderValue::from(meta.content_length())); + headers.insert("x-reduct-time", HeaderValue::from(meta.timestamp())); + headers.insert("x-reduct-last", HeaderValue::from(i64::from(meta.last()))); headers }; diff --git a/reductstore/src/api/entry/update_batched.rs b/reductstore/src/api/entry/update_batched.rs index 0b651a61c..7a7963678 100644 --- a/reductstore/src/api/entry/update_batched.rs +++ b/reductstore/src/api/entry/update_batched.rs @@ -100,7 +100,7 @@ mod tests { use axum::response::IntoResponse; use axum_extra::headers::HeaderValue; use bytes::Bytes; - use reduct_base::io::RecordMeta; + use reduct_base::io::ReadRecord; use rstest::rstest; #[rstest] @@ -192,9 +192,9 @@ mod tests { .begin_read(0) .await .unwrap(); - assert_eq!(reader.labels().len(), 2); - assert_eq!(&reader.labels()["x"], "z"); - assert_eq!(&reader.labels()["1"], "2"); + assert_eq!(reader.meta().labels().len(), 2); + assert_eq!(&reader.meta().labels()["x"], "z"); + assert_eq!(&reader.meta().labels()["1"], "2"); } assert_eq!(err_map.len(), 0); diff --git a/reductstore/src/api/entry/update_single.rs b/reductstore/src/api/entry/update_single.rs index 929fdd58f..6b9a68a17 100644 --- a/reductstore/src/api/entry/update_single.rs +++ b/reductstore/src/api/entry/update_single.rs @@ -93,7 +93,7 @@ mod tests { use crate::api::tests::{components, empty_body, path_to_entry_1}; use axum_extra::headers::{Authorization, HeaderMapExt}; - use reduct_base::io::RecordMeta; + use reduct_base::io::ReadRecord; use rstest::*; use super::*; @@ -133,9 +133,9 @@ mod tests { .await .unwrap(); - assert_eq!(record.labels().len(), 2); - assert_eq!(&record.labels()["x"], "z",); - assert_eq!(&record.labels()["1"], "2",); + assert_eq!(record.meta().labels().len(), 2); + assert_eq!(&record.meta().labels()["x"], "z",); + assert_eq!(&record.meta().labels()["1"], "2",); let info = components .replication_repo diff --git a/reductstore/src/api/entry/write_batched.rs b/reductstore/src/api/entry/write_batched.rs index bd217b904..2454db2d7 100644 --- a/reductstore/src/api/entry/write_batched.rs +++ b/reductstore/src/api/entry/write_batched.rs @@ -277,7 +277,7 @@ mod tests { use axum_extra::headers::HeaderValue; use reduct_base::error::ErrorCode; - use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::io::ReadRecord; use rstest::{fixture, rstest}; #[rstest] @@ -379,9 +379,9 @@ mod tests { .begin_read(1) .await .unwrap(); - assert_eq!(&reader.labels()["a"], "b"); - assert_eq!(reader.content_type(), "text/plain"); - assert_eq!(reader.content_length(), 10); + assert_eq!(&reader.meta().labels()["a"], "b"); + assert_eq!(reader.meta().content_type(), "text/plain"); + assert_eq!(reader.meta().content_length(), 10); assert_eq!(reader.read().await.unwrap(), Ok(Bytes::from("1234567890"))); } { @@ -392,9 +392,9 @@ mod tests { .begin_read(2) .await .unwrap(); - assert_eq!(&reader.labels()["c"], "d,f"); - assert_eq!(reader.content_type(), "text/plain"); - assert_eq!(reader.content_length(), 20); + assert_eq!(&reader.meta().labels()["c"], "d,f"); + assert_eq!(reader.meta().content_type(), "text/plain"); + assert_eq!(reader.meta().content_length(), 20); assert_eq!( reader.read().await.unwrap(), Ok(Bytes::from("abcdef1234567890abcd")) @@ -408,9 +408,9 @@ mod tests { .begin_read(10) .await .unwrap(); - assert!(reader.labels().is_empty()); - assert_eq!(reader.content_type(), "text/plain"); - assert_eq!(reader.content_length(), 18); + assert!(reader.meta().labels().is_empty()); + assert_eq!(reader.meta().content_type(), "text/plain"); + assert_eq!(reader.meta().content_length(), 18); assert_eq!( reader.read().await.unwrap(), Ok(Bytes::from("ef1234567890abcdef")) @@ -482,12 +482,12 @@ mod tests { .upgrade_and_unwrap(); { let mut reader = bucket.begin_read("entry-1", 1).await.unwrap(); - assert_eq!(reader.content_length(), 10); + assert_eq!(reader.meta().content_length(), 10); assert_eq!(reader.read().await.unwrap(), Ok(Bytes::from("1234567890"))); } { let mut reader = bucket.begin_read("entry-1", 3).await.unwrap(); - assert_eq!(reader.content_length(), 18); + assert_eq!(reader.meta().content_length(), 18); assert_eq!( reader.read().await.unwrap(), Ok(Bytes::from("ef1234567890abcdef")) diff --git a/reductstore/src/api/entry/write_single.rs b/reductstore/src/api/entry/write_single.rs index 361e2b180..67d4c1a84 100644 --- a/reductstore/src/api/entry/write_single.rs +++ b/reductstore/src/api/entry/write_single.rs @@ -141,7 +141,7 @@ mod tests { use crate::api::tests::{components, empty_body, path_to_entry_1}; use axum_extra::headers::{Authorization, HeaderMapExt}; - use reduct_base::io::RecordMeta; + use reduct_base::io::ReadRecord; use reduct_base::not_found; use rstest::*; @@ -176,7 +176,7 @@ mod tests { .await .unwrap(); - assert_eq!(&record.labels()["x"], "y"); + assert_eq!(&record.meta().labels()["x"], "y"); let info = components .replication_repo diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index 75f631631..36ed8d3ba 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -1,6 +1,9 @@ // Copyright 2025 ReductSoftware UG // Licensed under the Business Source License 1.1 +mod create; +mod load; + use crate::asset::asset_manager::ManageStaticAsset; use crate::ext::filter::ExtWhenFilter; use crate::storage::query::base::QueryOptions; @@ -8,13 +11,16 @@ use crate::storage::query::condition::{EvaluationStage, Parser}; use crate::storage::query::QueryRx; use async_trait::async_trait; use dlopen2::wrapper::{Container, WrapperApi}; -use log::{error, info}; +use futures_util::StreamExt; +use reduct_base::error::ErrorCode::NoContent; use reduct_base::error::ReductError; -use reduct_base::ext::{BoxedReadRecord, ExtSettings, IoExtension, ProcessStatus}; +use reduct_base::ext::{ + BoxedCommiter, BoxedProcessor, BoxedReadRecord, BoxedRecordStream, ExtSettings, IoExtension, +}; use reduct_base::msg::entry_api::QueryEntry; -use reduct_base::{internal_server_error, unprocessable_entity, Labels}; +use reduct_base::{no_content, unprocessable_entity}; use std::collections::HashMap; -use std::path::PathBuf; +use std::pin::Pin; use std::sync::Arc; use std::time::Instant; use tokio::sync::RwLock as AsyncRwLock; @@ -36,21 +42,36 @@ pub(crate) trait ManageExtensions { entry_name: &str, query: QueryEntry, ) -> Result<(), ReductError>; - async fn next_processed_record( + + /// Fetches and processes a record from the extension. + /// + /// This method is called for each record that is fetched from the storage engine. + /// + /// # Arguments + /// + /// * `query_id` - The ID of the query. + /// * `query_rx` - The receiver for the query. + /// + /// # Returns + /// + /// 204 No Content if no records are available. + async fn fetch_and_process_record( &self, query_id: u64, query_rx: Arc>, - ) -> ProcessStatus; + ) -> Option>; } pub type BoxedManageExtensions = Box; pub(crate) struct QueryContext { - id: u64, query: QueryOptions, condition_filter: ExtWhenFilter, last_access: Instant, - ext_pipeline: Vec, + current_stream: Option>, + + processor: BoxedProcessor, + commiter: BoxedCommiter, } struct ExtRepository { @@ -64,93 +85,6 @@ struct ExtRepository { embedded_extensions: Vec>, // we need to keep them from being cleaned up } -impl ExtRepository { - fn try_load( - paths: Vec, - embedded_extensions: Vec>, - settings: ExtSettings, - ) -> Result { - let mut extension_map = IoExtMap::new(); - - let query_map = AsyncRwLock::new(HashMap::new()); - let mut ext_wrappers = Vec::new(); - - for path in paths { - if !path.exists() { - return Err(internal_server_error!( - "Extension directory {:?} does not exist", - path - )); - } - - for entry in path.read_dir()? { - let path = entry?.path(); - if path.is_file() - && path - .extension() - .map_or(false, |ext| ext == "so" || ext == "dll" || ext == "dylib") - { - let ext_wrapper = unsafe { - match Container::::load(path.clone()) { - Ok(wrapper) => wrapper, - Err(e) => { - error!("Failed to load extension '{:?}': {:?}", path, e); - continue; - } - } - }; - - let ext = unsafe { Box::from_raw(ext_wrapper.get_ext(settings.clone())) }; - - info!("Load extension: {:?}", ext.info()); - - let name = ext.info().name().to_string(); - extension_map.insert(name, Arc::new(AsyncRwLock::new(ext))); - ext_wrappers.push(ext_wrapper); - } - } - } - - Ok(ExtRepository { - extension_map, - query_map, - ext_wrappers, - embedded_extensions, - }) - } - - async fn send_record_to_ext_pipeline( - query_id: u64, - mut record: BoxedReadRecord, - ext_pipeline: Vec, - ) -> ProcessStatus { - let mut computed_labels = Labels::new(); - for ext in ext_pipeline { - computed_labels.extend(record.computed_labels().clone().into_iter()); - let status = ext - .write() - .await - .next_processed_record(query_id, record) - .await; - - if let ProcessStatus::Ready(result) = status { - if let Ok(processed_record) = result { - record = processed_record; - } else { - return ProcessStatus::Ready(result); - } - } else { - return status; - } - } - - record - .computed_labels_mut() - .extend(computed_labels.clone().into_iter()); - ProcessStatus::Ready(Ok(record)) - } -} - #[async_trait] impl ManageExtensions for ExtRepository { /// Register a query with the extension @@ -172,29 +106,40 @@ impl ManageExtensions for ExtRepository { entry_name: &str, query_request: QueryEntry, ) -> Result<(), ReductError> { - let mut pipeline = Vec::new(); let mut query_map = self.query_map.write().await; - let ext_params = query_request.ext.as_ref(); - if ext_params.is_some() && ext_params.unwrap().is_object() { - for (name, _) in ext_params.unwrap().as_object().unwrap().iter() { - if let Some(ext) = self.extension_map.get(name) { - ext.write().await.register_query( - query_id, - bucket_name, - entry_name, - &query_request, - )?; - pipeline.push(Arc::clone(ext)); - } else { - return Err(unprocessable_entity!( - "Unknown extension '{}' in query id={}", - name, - query_id - )); - } + let controllers = if ext_params.is_some() && ext_params.unwrap().is_object() { + let ext_query = ext_params.unwrap().as_object().unwrap(); + if ext_query.iter().count() > 1 { + return Err(unprocessable_entity!( + "Multiple extensions are not supported in query id={}", + query_id + )); } - } + + let Some(name) = ext_query.keys().next() else { + return Err(unprocessable_entity!( + "Extension name is not found in query id={}", + query_id + )); + }; + + if let Some(ext) = self.extension_map.get(name) { + let (processor, commiter) = + ext.write() + .await + .query(bucket_name, entry_name, &query_request)?; + Some((processor, commiter)) + } else { + return Err(unprocessable_entity!( + "Unknown extension '{}' in query id={}", + name, + query_id + )); + } + } else { + None + }; let query_options: QueryOptions = query_request.into(); // remove expired queries @@ -202,11 +147,6 @@ impl ManageExtensions for ExtRepository { for (key, query) in query_map.iter() { if query.last_access.elapsed() > query.query.ttl { - for ext in &query.ext_pipeline { - if let Err(e) = ext.write().await.unregister_query(query.id) { - error!("Failed to unregister query {}: {:?}", query_id, e); - } - } ids_to_remove.push(*key); } } @@ -218,7 +158,7 @@ impl ManageExtensions for ExtRepository { // check if the query has references to computed labels and no extension is found let condition = if let Some(condition) = &query_options.when { let node = Parser::new().parse(condition)?; - if pipeline.is_empty() && node.stage() == &EvaluationStage::Compute { + if controllers.is_none() && node.stage() == &EvaluationStage::Compute { return Err(unprocessable_entity!( "There is at least one reference to computed labels but no extension is found" )); @@ -228,130 +168,112 @@ impl ManageExtensions for ExtRepository { None }; - if pipeline.is_empty() { - // No extension found, we don't need to register the query - return Ok(()); + let condition_filter = ExtWhenFilter::new(condition, query_options.strict); + + if let Some((processor, commiter)) = controllers { + query_map.insert(query_id, { + QueryContext { + query: query_options, + condition_filter, + last_access: Instant::now(), + current_stream: None, + processor, + commiter, + } + }); } - query_map.insert(query_id, { - QueryContext { - id: query_id, - query: query_options, - condition_filter: ExtWhenFilter::new(condition), - last_access: Instant::now(), - ext_pipeline: pipeline, - } - }); - Ok(()) } - async fn next_processed_record( + async fn fetch_and_process_record( &self, query_id: u64, query_rx: Arc>, - ) -> ProcessStatus { - // check if registered - if let Some(record) = query_rx.write().await.recv().await { - match record { - Ok(record) => { - let record = Box::new(record); - let pipline = { - // check if query is registered and - let mut lock = self.query_map.write().await; - - let query = lock.get_mut(&query_id); - if query.is_none() { - return ProcessStatus::Ready(Ok(record)); - } - - let query = query.unwrap(); - query.last_access = Instant::now(); - query.ext_pipeline.clone() - }; - - let status = Self::send_record_to_ext_pipeline(query_id, record, pipline).await; - if let ProcessStatus::Ready(Ok(record)) = status { - let mut lock = self.query_map.write().await; - let query = lock.get_mut(&query_id).unwrap(); - query - .condition_filter - .filter_record(ProcessStatus::Ready(Ok(record)), query.query.strict) - } else { - status - } - } - Err(e) => ProcessStatus::Ready(Err(e)), + ) -> Option> { + // TODO: The code is awkward, we need to refactor it + // unfortunately stream! macro does not work here and crashes compiler + let mut lock = self.query_map.write().await; + let query = match lock.get_mut(&query_id) { + Some(query) => query, + None => { + return query_rx + .write() + .await + .recv() + .await + .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)) } - } else { - ProcessStatus::Stop - } - } -} - -pub fn create_ext_repository( - external_path: Option, - embedded_extensions: Vec>, - settings: ExtSettings, -) -> Result { - if external_path.is_some() || !embedded_extensions.is_empty() { - let mut paths = if let Some(path) = external_path { - vec![path] - } else { - Vec::new() }; - for embedded in &embedded_extensions { - if let Ok(path) = embedded.absolut_path("") { - paths.push(path); - } - } + query.last_access = Instant::now(); - Ok(Box::new(ExtRepository::try_load( - paths, - embedded_extensions, - settings, - )?)) - } else { - // Dummy extension repository if - struct NoExtRepository; + if let Some(mut current_stream) = query.current_stream.take() { + let item = current_stream.next().await; + query.current_stream = Some(current_stream); - #[async_trait] - impl ManageExtensions for NoExtRepository { - async fn register_query( - &self, - _query_id: u64, - _bucket_name: &str, - _entry_name: &str, - _query: QueryEntry, - ) -> Result<(), ReductError> { - Ok(()) + if let Some(result) = item { + if let Err(e) = result { + return Some(Err(e)); + } + + let record = result.unwrap(); + + return match query.condition_filter.filter_record(record) { + Some(result) => { + let record = match result { + Ok(record) => record, + Err(e) => return Some(Err(e)), + }; + + query.commiter.commit_record(record).await + } + None => None, + }; + } else { + // stream is empty, we need to process the next record + query.current_stream = None; } + } - async fn next_processed_record( - &self, - _query_id: u64, - query_rx: Arc>, - ) -> ProcessStatus { - let record = query_rx.write().await.recv().await; - match record { - Some(Ok(record)) => ProcessStatus::Ready(Ok(Box::new(record))), - Some(Err(e)) => ProcessStatus::Ready(Err(e)), - None => ProcessStatus::Stop, + let Some(record) = query_rx.write().await.recv().await else { + return Some(Err(no_content!("No content"))); + }; + + let record = match record { + Ok(record) => record, + Err(e) => { + return if e.status == NoContent { + if let Some(last_record) = query.commiter.flush().await { + Some(last_record) + } else { + Some(Err(e)) + } + } else { + Some(Err(e)) } } - } + }; - Ok(Box::new(NoExtRepository)) + assert!(query.current_stream.is_none(), "Must be None"); + + let stream = match query.processor.process_record(Box::new(record)).await { + Ok(stream) => stream, + Err(e) => return Some(Err(e)), + }; + + query.current_stream = Some(Box::into_pin(stream)); + None } } +pub(crate) use create::create_ext_repository; + #[cfg(test)] pub(super) mod tests { use super::*; - use reduct_base::ext::IoExtensionInfo; - use reqwest::blocking::get; - use reqwest::StatusCode; + use futures_util::Stream; + use reduct_base::ext::{Commiter, IoExtensionInfo, Processor}; use rstest::{fixture, rstest}; use crate::storage::entry::RecordReader; @@ -361,106 +283,14 @@ pub(super) mod tests { use prost_wkt_types::Timestamp; use reduct_base::io::{ReadChunk, ReadRecord, RecordMeta}; use reduct_base::msg::server_api::ServerInfo; + use reduct_base::Labels; use serde_json::json; - use std::fs; + use std::task::{Context, Poll}; use tempfile::tempdir; - use test_log::test as log_test; - - mod load { - use super::*; - use reduct_base::msg::server_api::ServerInfo; - #[log_test(rstest)] - fn test_load_extension(ext_repo: ExtRepository) { - assert_eq!(ext_repo.extension_map.len(), 1); - let ext = ext_repo - .extension_map - .get("test-ext") - .unwrap() - .blocking_read(); - let info = ext.info().clone(); - assert_eq!( - info, - IoExtensionInfo::builder() - .name("test-ext") - .version("0.1.1") - .build() - ); - } - - #[log_test(rstest)] - fn test_failed_load(ext_settings: ExtSettings) { - let path = tempdir().unwrap().keep(); - fs::create_dir_all(&path).unwrap(); - fs::write(&path.join("libtest.so"), b"test").unwrap(); - let ext_repo = ExtRepository::try_load(vec![path], vec![], ext_settings).unwrap(); - assert_eq!(ext_repo.extension_map.len(), 0); - } - - #[log_test(rstest)] - fn test_failed_open_dir(ext_settings: ExtSettings) { - let path = PathBuf::from("non_existing_dir"); - let ext_repo = ExtRepository::try_load(vec![path], vec![], ext_settings); - assert_eq!( - ext_repo.err().unwrap(), - internal_server_error!("Extension directory \"non_existing_dir\" does not exist") - ); - } - - #[fixture] - fn ext_settings() -> ExtSettings { - ExtSettings::builder() - .server_info(ServerInfo::default()) - .build() - } - #[fixture] - fn ext_repo(ext_settings: ExtSettings) -> ExtRepository { - // This is the path to the build directory of the extension from ext_stub crate - const EXTENSION_VERSION: &str = "0.1.1"; - - if !cfg!(target_arch = "x86_64") { - panic!("Unsupported architecture"); - } - - let file_name = if cfg!(target_os = "linux") { - // This is the path to the build directory of the extension from ext_stub crate - "libtest_ext-x86_64-unknown-linux-gnu.so" - } else if cfg!(target_os = "macos") { - "libtest_ext-x86_64-apple-darwin.dylib" - } else if cfg!(target_os = "windows") { - "libtest_ext-x86_64-pc-windows-gnu.dll" - } else { - panic!("Unsupported platform") - }; - - let ext_path = PathBuf::from(tempdir().unwrap().keep()).join("ext"); - fs::create_dir_all(ext_path.clone()).unwrap(); - - let link = format!( - "https://github.com/reductstore/test-ext/releases/download/v{}/{}", - EXTENSION_VERSION, file_name - ); - - let mut resp = get(link).expect("Failed to download extension"); - if resp.status() != StatusCode::OK { - if resp.status() == StatusCode::FOUND { - resp = get(resp.headers().get("location").unwrap().to_str().unwrap()) - .expect("Failed to download extension"); - } else { - panic!("Failed to download extension: {}", resp.status()); - } - } - - fs::write(ext_path.join(file_name), resp.bytes().unwrap()) - .expect("Failed to write extension"); - - let empty_ext_path = tempdir().unwrap().keep(); - ExtRepository::try_load(vec![ext_path, empty_ext_path], vec![], ext_settings).unwrap() - } - } mod register_query { use super::*; - use mockall::predicate::ge; + use std::time::Duration; #[rstest] @@ -482,7 +312,11 @@ pub(super) mod tests { #[rstest] #[tokio::test] - async fn test_with_ext_part(mut mock_ext: MockIoExtension) { + async fn test_with_ext_part( + mut mock_ext: MockIoExtension, + processor: BoxedProcessor, + commiter: BoxedCommiter, + ) { let query = QueryEntry { ext: Some(json!({ "test-ext": {}, @@ -491,9 +325,9 @@ pub(super) mod tests { }; mock_ext - .expect_register_query() - .with(eq(1), eq("bucket"), eq("entry"), eq(query.clone())) - .return_const(Ok(())); + .expect_query() + .with(eq("bucket"), eq("entry"), eq(query.clone())) + .return_once(|_, _, _| Ok((processor, commiter))); let mocked_ext_repo = mocked_ext_repo("test-ext", mock_ext); @@ -517,7 +351,7 @@ pub(super) mod tests { when: Some(json!({"@label": { "$eq": "value" }})), ..Default::default() }; - mock_ext.expect_register_query().never(); + mock_ext.expect_query().never(); let mocked_ext_repo = mocked_ext_repo("test", mock_ext); @@ -545,17 +379,15 @@ pub(super) mod tests { }; mock_ext - .expect_register_query() - .with(ge(1), eq("bucket"), eq("entry"), eq(query.clone())) - .return_const(Ok(())); - mock_ext - .expect_unregister_query() - .with(eq(1)) - .return_const(Ok(())); - mock_ext - .expect_unregister_query() - .with(eq(2)) - .return_const(Ok(())); + .expect_query() + .with(eq("bucket"), eq("entry"), eq(query.clone())) + .returning(|_, _, _| { + Ok(( + Box::new(MockProcessor::new()), + Box::new(MockCommiter::new()), + )) + }) + .times(3); let mocked_ext_repo = mocked_ext_repo("test-ext", mock_ext); assert!(mocked_ext_repo @@ -589,16 +421,26 @@ pub(super) mod tests { } #[rstest] + #[case(json!({"test-ext": {}, "test-ext2": {}}), unprocessable_entity!("Multiple extensions are not supported in query id=1"))] + #[case(json!({"unknown-ext": {}}), unprocessable_entity!("Unknown extension 'unknown-ext' in query id=1"))] + #[case(json!({}), unprocessable_entity!("Extension name is not found in query id=1"))] #[tokio::test] - async fn error_unknown_extension(mut mock_ext: MockIoExtension) { + async fn test_error_handling( + mut mock_ext: MockIoExtension, + processor: BoxedProcessor, + commiter: BoxedCommiter, + #[case] ext_params: serde_json::Value, + #[case] expected_error: ReductError, + ) { let query = QueryEntry { - ext: Some(json!({ - "unknown-ext": {}, - })), + ext: Some(ext_params), ..Default::default() }; - mock_ext.expect_register_query().never(); + mock_ext + .expect_query() + .with(eq("bucket"), eq("entry"), eq(query.clone())) + .return_once(|_, _, _| Ok((processor, commiter))); let mocked_ext_repo = mocked_ext_repo("test-ext", mock_ext); assert_eq!( @@ -607,79 +449,15 @@ pub(super) mod tests { .await .err() .unwrap(), - unprocessable_entity!("Unknown extension 'unknown-ext' in query id=1") + expected_error ); } } - mod send_record_to_ext_pipeline { - use super::*; - use assert_matches::assert_matches; - use mockall::predicate::always; - use reduct_base::internal_server_error; - - #[rstest] - #[tokio::test] - async fn test_no_ext() { - let record = Box::new(MockRecord::new("key1", "val1")); - let query = QueryContext { - id: 1, - query: QueryOptions::default(), - condition_filter: ExtWhenFilter::new(None), - last_access: Instant::now(), - ext_pipeline: vec![], - }; - - let status = ExtRepository::send_record_to_ext_pipeline( - query.id, - record, - query.ext_pipeline.clone(), - ) - .await; - assert_matches!(status, ProcessStatus::Ready(_)); - } - - #[rstest] - #[tokio::test(flavor = "current_thread")] - async fn test_stop_pipeline_at_errors() { - let record = Box::new(MockRecord::new("key1", "val1")); - let mut mock_ext_1 = MockIoExtension::new(); - let mut mock_ext_2 = MockIoExtension::new(); - - mock_ext_1 - .expect_next_processed_record() - .with(eq(1), always()) - .return_once(|_, _| ProcessStatus::Ready(Err(internal_server_error!("test")))); - mock_ext_2.expect_next_processed_record().never(); - - let query = QueryContext { - id: 1, - query: QueryOptions::default(), - condition_filter: ExtWhenFilter::new(None), - last_access: Instant::now(), - ext_pipeline: vec![ - Arc::new(AsyncRwLock::new(Box::new(mock_ext_1))), - Arc::new(AsyncRwLock::new(Box::new(mock_ext_2))), - ], - }; - - let status = ExtRepository::send_record_to_ext_pipeline( - query.id, - record, - query.ext_pipeline.clone(), - ) - .await; - let ProcessStatus::Ready(result) = status else { - panic!("Expected ProcessStatus::Ready"); - }; - - assert_eq!(result.err().unwrap(), internal_server_error!("test")); - } - } mod next_processed_record { use super::*; use crate::storage::entry::RecordReader; - use assert_matches::assert_matches; + use mockall::predicate; use reduct_base::internal_server_error; @@ -691,10 +469,10 @@ pub(super) mod tests { drop(tx); let query_rx = Arc::new(AsyncRwLock::new(rx)); - assert_matches!( - mocked_ext_repo.next_processed_record(1, query_rx).await, - ProcessStatus::Stop - ); + assert!(mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .is_none(),); } #[rstest] @@ -706,9 +484,14 @@ pub(super) mod tests { tx.send(Err(err.clone())).await.unwrap(); let query_rx = Arc::new(AsyncRwLock::new(rx)); - assert_matches!( - mocked_ext_repo.next_processed_record(1, query_rx).await, - ProcessStatus::Ready(Err(_)) + assert_eq!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .unwrap() + .err() + .unwrap(), + err ); } @@ -721,10 +504,11 @@ pub(super) mod tests { tx.send(Ok(record_reader)).await.unwrap(); let query_rx = Arc::new(AsyncRwLock::new(rx)); - assert_matches!( - mocked_ext_repo.next_processed_record(1, query_rx).await, - ProcessStatus::Ready(Ok(_)) - ); + assert!(mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .unwrap() + .is_ok(),); } #[rstest] @@ -732,12 +516,18 @@ pub(super) mod tests { async fn test_process_not_ready( record_reader: RecordReader, mut mock_ext: MockIoExtension, + mut processor: Box, + mut commiter: Box, ) { - mock_ext.expect_register_query().return_const(Ok(())); + processor + .expect_process_record() + .return_once(|_| Ok(MockStream::boxed(Poll::Pending) as BoxedRecordStream)); + commiter.expect_commit_record().never(); + mock_ext - .expect_next_processed_record() - .with(eq(1), predicate::always()) - .return_once(|_, _| ProcessStatus::NotReady); + .expect_query() + .with(eq("bucket"), eq("entry"), predicate::always()) + .return_once(|_, _, _| Ok((processor, commiter))); let query = QueryEntry { ext: Some(json!({ @@ -757,71 +547,167 @@ pub(super) mod tests { tx.send(Ok(record_reader)).await.unwrap(); let query_rx = Arc::new(AsyncRwLock::new(rx)); - let status = mocked_ext_repo.next_processed_record(1, query_rx).await; - let ProcessStatus::NotReady = status else { - panic!("Expected ProcessStatus::NotReady"); - }; + assert!(mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .is_none()); } #[rstest] #[tokio::test(flavor = "current_thread")] - async fn test_process_in_pipeline(record_reader: RecordReader) { - let record1 = Box::new(MockRecord::new("key1", "val1")); - let record2 = Box::new(MockRecord::new("key2", "val2")); - - let mut mock1 = MockIoExtension::new(); - let mut mock2 = MockIoExtension::new(); - - mock1.expect_register_query().return_const(Ok(())); - mock2.expect_register_query().return_const(Ok(())); - mock1 - .expect_next_processed_record() - .with(eq(1), predicate::always()) - .return_once(|_, _| ProcessStatus::Ready(Ok(record1))); - mock2 - .expect_next_processed_record() - .with(eq(1), predicate::always()) - .return_once(|_, _| ProcessStatus::Ready(Ok(record2))); - - let mut mocked_ext_repo = mocked_ext_repo("test1", mock1); - mocked_ext_repo.extension_map.insert( - "test2".to_string(), - Arc::new(AsyncRwLock::new(Box::new(mock2))), - ); + async fn test_process_a_record( + record_reader: RecordReader, + mut mock_ext: MockIoExtension, + mut processor: Box, + mut commiter: Box, + ) { + processor.expect_process_record().return_once(|_| { + Ok(MockStream::boxed(Poll::Ready(Some(Ok(MockRecord::boxed( + "key", "val", + )))))) + }); + + commiter.expect_commit_record().return_once(|_| { + Some(Ok( + Box::new(MockRecord::new("key", "val")) as BoxedReadRecord + )) + }); + commiter.expect_flush().return_once(|| None).times(1); + + mock_ext + .expect_query() + .with(eq("bucket"), eq("entry"), predicate::always()) + .return_once(|_, _, _| Ok((processor, commiter))); let query = QueryEntry { ext: Some(json!({ "test1": {}, - "test2": {} })), - when: Some(json!({"@key2": { "$eq": "val2" }})), ..Default::default() }; + + let mocked_ext_repo = mocked_ext_repo("test1", mock_ext); + mocked_ext_repo .register_query(1, "bucket", "entry", query) .await .unwrap(); - let (tx, rx) = tokio::sync::mpsc::channel(1); + let (tx, rx) = tokio::sync::mpsc::channel(2); tx.send(Ok(record_reader)).await.unwrap(); + tx.send(Err(no_content!(""))).await.unwrap(); let query_rx = Arc::new(AsyncRwLock::new(rx)); - let status = mocked_ext_repo.next_processed_record(1, query_rx).await; - let ProcessStatus::Ready(record) = status else { - panic!("Expected ProcessStatus::Ready"); + assert!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .is_none(), + "First run should be None (stupid implementation)" + ); + + let record = mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .unwrap(); + + assert_eq!(record.unwrap().read().await, None); + + assert_eq!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .unwrap() + .err() + .unwrap(), + no_content!("") + ); + } + + #[rstest] + #[tokio::test(flavor = "current_thread")] + async fn test_process_flushed_record( + record_reader: RecordReader, + mut mock_ext: MockIoExtension, + mut processor: Box, + mut commiter: Box, + ) { + processor.expect_process_record().return_once(|_| { + Ok(MockStream::boxed(Poll::Ready(Some(Ok(MockRecord::boxed( + "key", "val", + )))))) + }); + + commiter.expect_commit_record().return_once(|_| None); + + commiter + .expect_flush() + .return_once(|| { + Some(Ok( + Box::new(MockRecord::new("key", "val")) as BoxedReadRecord + )) + }) + .times(1); + + mock_ext + .expect_query() + .with(eq("bucket"), eq("entry"), predicate::always()) + .return_once(|_, _, _| Ok((processor, commiter))); + + let query = QueryEntry { + ext: Some(json!({ + "test1": {}, + })), + ..Default::default() }; + let mocked_ext_repo = mocked_ext_repo("test1", mock_ext); + mocked_ext_repo + .register_query(1, "bucket", "entry", query) + .await + .unwrap(); + + let (tx, rx) = tokio::sync::mpsc::channel(2); + tx.send(Ok(record_reader)).await.unwrap(); + tx.send(Err(no_content!(""))).await.unwrap(); + + let query_rx = Arc::new(AsyncRwLock::new(rx)); + assert!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .is_none(), + "First run should be None (stupid implementation)" + ); + + assert!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .is_none(), + "we don't commit the record waiting for flush" + ); + + assert!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .unwrap() + .is_ok(), + "we should get the record from flush" + ); + + drop(tx); // close the channel to simulate no more records assert_eq!( - record.unwrap().computed_labels(), - &Labels::from_iter( - vec![ - ("key1".to_string(), "val1".to_string()), - ("key2".to_string(), "val2".to_string()) - ] - .into_iter() - ), - "Computed labels to be merged from both extensions" + mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .unwrap() + .err() + .unwrap(), + no_content!("No content"), + "and we are done" ); } } @@ -831,6 +717,16 @@ pub(super) mod tests { MockIoExtension::new() } + #[fixture] + fn processor() -> Box { + Box::new(MockProcessor::new()) + } + + #[fixture] + fn commiter() -> Box { + Box::new(MockCommiter::new()) + } + #[fixture] fn record_reader() -> RecordReader { let record = Record { @@ -868,53 +764,84 @@ pub(super) mod tests { impl IoExtension for IoExtension { fn info(&self) -> &IoExtensionInfo; - fn register_query( + + fn query( &mut self, - query_id: u64, bucket_name: &str, entry_name: &str, query: &QueryEntry, - ) -> Result<(), ReductError>; + ) -> Result<(BoxedProcessor, BoxedCommiter), ReductError>; + } + + } - fn unregister_query(&mut self, query_id: u64) -> Result<(), ReductError>; + mock! { + Processor {} - async fn next_processed_record( + #[async_trait] + impl Processor for Processor { + async fn process_record( &mut self, - query_id: u64, record: BoxedReadRecord, - ) -> ProcessStatus; + ) -> Result; + } + } + + mock! { + Commiter {} + + #[async_trait] + impl Commiter for Commiter { + async fn commit_record(&mut self, record: BoxedReadRecord) -> Option>; + async fn flush(&mut self) -> Option>; + } + } + + struct MockStream { + ret_value: Option>>>, + } + impl Stream for MockStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { + let Some(ret_value) = self.ret_value.take() else { + return Poll::Ready(None); + }; + + ret_value } + } + impl MockStream { + fn boxed(ret_value: Poll>>) -> Box { + Box::new(MockStream { + ret_value: Some(ret_value), + }) + } } #[derive(Clone, PartialEq, Debug)] pub struct MockRecord { - computed_labels: Labels, - labels: Labels, + meta: RecordMeta, } impl MockRecord { pub fn new(key: &str, val: &str) -> Self { - MockRecord { - computed_labels: Labels::from_iter( + let meta = RecordMeta::builder() + .timestamp(0) + .computed_labels(Labels::from_iter( vec![(key.to_string(), val.to_string())].into_iter(), - ), - labels: Labels::new(), - } - } - - pub fn labels_mut(&mut self) -> &mut Labels { - &mut self.labels + )) + .build(); + MockRecord { meta } } - } - impl RecordMeta for MockRecord { - fn timestamp(&self) -> u64 { - 0 + pub fn meta_mut(&mut self) -> &mut RecordMeta { + &mut self.meta } - fn labels(&self) -> &Labels { - &self.labels + pub fn boxed(key: &str, val: &str) -> BoxedReadRecord { + Box::new(MockRecord::new(key, val)) } } @@ -924,24 +851,8 @@ pub(super) mod tests { None } - fn last(&self) -> bool { - todo!() - } - - fn computed_labels(&self) -> &Labels { - &self.computed_labels - } - - fn computed_labels_mut(&mut self) -> &mut Labels { - &mut self.computed_labels - } - - fn content_length(&self) -> u64 { - todo!() - } - - fn content_type(&self) -> &str { - todo!() + fn meta(&self) -> &RecordMeta { + &self.meta } } } diff --git a/reductstore/src/ext/ext_repository/create.rs b/reductstore/src/ext/ext_repository/create.rs new file mode 100644 index 000000000..2e8e350ae --- /dev/null +++ b/reductstore/src/ext/ext_repository/create.rs @@ -0,0 +1,69 @@ +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 + +use crate::asset::asset_manager::ManageStaticAsset; +use crate::ext::ext_repository::{BoxedManageExtensions, ExtRepository, ManageExtensions}; +use crate::storage::query::QueryRx; +use async_trait::async_trait; +use reduct_base::error::ReductError; +use reduct_base::ext::{BoxedReadRecord, ExtSettings}; +use reduct_base::msg::entry_api::QueryEntry; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::RwLock as AsyncRwLock; +pub fn create_ext_repository( + external_path: Option, + embedded_extensions: Vec>, + settings: ExtSettings, +) -> Result { + if external_path.is_some() || !embedded_extensions.is_empty() { + let mut paths = if let Some(path) = external_path { + vec![path] + } else { + Vec::new() + }; + + for embedded in &embedded_extensions { + if let Ok(path) = embedded.absolut_path("") { + paths.push(path); + } + } + + Ok(Box::new(ExtRepository::try_load( + paths, + embedded_extensions, + settings, + )?)) + } else { + // Dummy extension repository if + struct NoExtRepository; + + #[async_trait] + impl ManageExtensions for NoExtRepository { + async fn register_query( + &self, + _query_id: u64, + _bucket_name: &str, + _entry_name: &str, + _query: QueryEntry, + ) -> Result<(), ReductError> { + Ok(()) + } + + async fn fetch_and_process_record( + &self, + _query_id: u64, + query_rx: Arc>, + ) -> Option> { + query_rx + .write() + .await + .recv() + .await + .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)) + } + } + + Ok(Box::new(NoExtRepository)) + } +} diff --git a/reductstore/src/ext/ext_repository/load.rs b/reductstore/src/ext/ext_repository/load.rs new file mode 100644 index 000000000..4f452741e --- /dev/null +++ b/reductstore/src/ext/ext_repository/load.rs @@ -0,0 +1,172 @@ +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 + +use crate::asset::asset_manager::ManageStaticAsset; +use crate::ext::ext_repository::{ExtRepository, ExtensionApi, IoExtMap}; +use dlopen2::wrapper::Container; +use log::{error, info}; +use reduct_base::error::ReductError; +use reduct_base::ext::ExtSettings; +use reduct_base::internal_server_error; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::RwLock as AsyncRwLock; + +impl ExtRepository { + pub(super) fn try_load( + paths: Vec, + embedded_extensions: Vec>, + settings: ExtSettings, + ) -> Result { + let mut extension_map = IoExtMap::new(); + + let query_map = AsyncRwLock::new(HashMap::new()); + let mut ext_wrappers = Vec::new(); + + for path in paths { + if !path.exists() { + return Err(internal_server_error!( + "Extension directory {:?} does not exist", + path + )); + } + + for entry in path.read_dir()? { + let path = entry?.path(); + if path.is_file() + && path + .extension() + .map_or(false, |ext| ext == "so" || ext == "dll" || ext == "dylib") + { + let ext_wrapper = unsafe { + match Container::::load(path.clone()) { + Ok(wrapper) => wrapper, + Err(e) => { + error!("Failed to load extension '{:?}': {:?}", path, e); + continue; + } + } + }; + + let ext = unsafe { Box::from_raw(ext_wrapper.get_ext(settings.clone())) }; + + info!("Load extension: {:?}", ext.info()); + + let name = ext.info().name().to_string(); + extension_map.insert(name, Arc::new(AsyncRwLock::new(ext))); + ext_wrappers.push(ext_wrapper); + } + } + } + + Ok(ExtRepository { + extension_map, + query_map, + ext_wrappers, + embedded_extensions, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reduct_base::ext::IoExtensionInfo; + use reduct_base::msg::server_api::ServerInfo; + use reqwest::blocking::get; + use reqwest::StatusCode; + use rstest::{fixture, rstest}; + use std::fs; + use tempfile::tempdir; + use test_log::test as log_test; + + #[log_test(rstest)] + fn test_load_extension(ext_repo: ExtRepository) { + assert_eq!(ext_repo.extension_map.len(), 1); + let ext = ext_repo + .extension_map + .get("test-ext") + .unwrap() + .blocking_read(); + let info = ext.info().clone(); + assert_eq!( + info, + IoExtensionInfo::builder() + .name("test-ext") + .version("0.1.1") + .build() + ); + } + + #[log_test(rstest)] + fn test_failed_load(ext_settings: ExtSettings) { + let path = tempdir().unwrap().keep(); + fs::create_dir_all(&path).unwrap(); + fs::write(&path.join("libtest.so"), b"test").unwrap(); + let ext_repo = ExtRepository::try_load(vec![path], vec![], ext_settings).unwrap(); + assert_eq!(ext_repo.extension_map.len(), 0); + } + + #[log_test(rstest)] + fn test_failed_open_dir(ext_settings: ExtSettings) { + let path = PathBuf::from("non_existing_dir"); + let ext_repo = ExtRepository::try_load(vec![path], vec![], ext_settings); + assert_eq!( + ext_repo.err().unwrap(), + internal_server_error!("Extension directory \"non_existing_dir\" does not exist") + ); + } + + #[fixture] + fn ext_settings() -> ExtSettings { + ExtSettings::builder() + .server_info(ServerInfo::default()) + .build() + } + + #[fixture] + fn ext_repo(ext_settings: ExtSettings) -> ExtRepository { + // This is the path to the build directory of the extension from ext_stub crate + const EXTENSION_VERSION: &str = "0.1.1"; + + if !cfg!(target_arch = "x86_64") { + panic!("Unsupported architecture"); + } + + let file_name = if cfg!(target_os = "linux") { + // This is the path to the build directory of the extension from ext_stub crate + "libtest_ext-x86_64-unknown-linux-gnu.so" + } else if cfg!(target_os = "macos") { + "libtest_ext-x86_64-apple-darwin.dylib" + } else if cfg!(target_os = "windows") { + "libtest_ext-x86_64-pc-windows-gnu.dll" + } else { + panic!("Unsupported platform") + }; + + let ext_path = PathBuf::from(tempdir().unwrap().keep()).join("ext"); + fs::create_dir_all(ext_path.clone()).unwrap(); + + let link = format!( + "https://github.com/reductstore/test-ext/releases/download/v{}/{}", + EXTENSION_VERSION, file_name + ); + + let mut resp = get(link).expect("Failed to download extension"); + if resp.status() != StatusCode::OK { + if resp.status() == StatusCode::FOUND { + resp = get(resp.headers().get("location").unwrap().to_str().unwrap()) + .expect("Failed to download extension"); + } else { + panic!("Failed to download extension: {}", resp.status()); + } + } + + fs::write(ext_path.join(file_name), resp.bytes().unwrap()) + .expect("Failed to write extension"); + + let empty_ext_path = tempdir().unwrap().keep(); + ExtRepository::try_load(vec![ext_path, empty_ext_path], vec![], ext_settings).unwrap() + } +} diff --git a/reductstore/src/ext/filter.rs b/reductstore/src/ext/filter.rs index efd980a71..92fda9299 100644 --- a/reductstore/src/ext/filter.rs +++ b/reductstore/src/ext/filter.rs @@ -4,11 +4,12 @@ use crate::storage::query::condition::{BoxedNode, Context, EvaluationStage}; use reduct_base::conflict; use reduct_base::error::ReductError; -use reduct_base::ext::{BoxedReadRecord, ProcessStatus}; +use reduct_base::ext::BoxedReadRecord; use std::collections::HashMap; pub(super) struct ExtWhenFilter { condition: Option, + strict: bool, } /// This filter is used to filter records based on a condition. @@ -16,47 +17,47 @@ pub(super) struct ExtWhenFilter { /// It is used in the `ext` module to filter records after they processed by extension, /// and it puts computed labels into the context. impl ExtWhenFilter { - pub fn new(condition: Option) -> Self { - ExtWhenFilter { condition } + pub fn new(condition: Option, strict: bool) -> Self { + ExtWhenFilter { condition, strict } } - pub fn filter_record(&mut self, status: ProcessStatus, strict: bool) -> ProcessStatus { + pub fn filter_record( + &mut self, + record: BoxedReadRecord, + ) -> Option> { if self.condition.is_none() { - return status; + return Some(Ok(record)); } // filter with computed labels - if let ProcessStatus::Ready(record) = &status { - match self.filter_with_computed(&record.as_ref().unwrap()) { - Ok(true) => status, - Ok(false) => ProcessStatus::NotReady, - Err(e) => { - if strict { - ProcessStatus::Ready(Err(e)) - } else { - status - } + match self.filter_with_computed(&record) { + Ok(true) => Some(Ok(record)), + Ok(false) => None, + Err(e) => { + if self.strict { + Some(Err(e)) + } else { + None } } - } else { - status } } fn filter_with_computed(&mut self, reader: &BoxedReadRecord) -> Result { - let mut labels = reader + let meta = reader.meta(); + let mut labels = meta .labels() .iter() .map(|(k, v)| (k.as_str(), v.as_str())) .collect::>(); - for (k, v) in reader.computed_labels() { + for (k, v) in meta.computed_labels() { if labels.insert(k, v).is_some() { return Err(conflict!("Computed label '@{}' already exists", k)); } } - let context = Context::new(reader.timestamp(), labels, EvaluationStage::Compute); + let context = Context::new(meta.timestamp(), labels, EvaluationStage::Compute); Ok(self .condition .as_mut() @@ -71,85 +72,96 @@ mod tests { use super::*; use crate::ext::ext_repository::tests::{mocked_record, MockRecord}; use crate::storage::query::condition::Parser; - use assert_matches::assert_matches; + + use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::Labels; use rstest::rstest; use serde_json::json; #[rstest] fn pass_status_if_condition_none(mocked_record: Box) { - let mut filter = ExtWhenFilter::new(None); - let status = ProcessStatus::Ready(Ok(mocked_record)); - assert_matches!( - filter.filter_record(status, false), - ProcessStatus::Ready(Ok(_)) - ) + let mut filter = ExtWhenFilter::new(None, false); + assert!(filter.filter_record(mocked_record).unwrap().is_ok()) } #[rstest] fn not_ready_if_condition_false(mocked_record: Box) { - let mut filter = ExtWhenFilter::new(Some( - Parser::new() - .parse(&json!({"$and": [false, "@key1"]})) - .unwrap(), - )); - let status = ProcessStatus::Ready(Ok(mocked_record)); - assert_matches!(filter.filter_record(status, true), ProcessStatus::NotReady) + let mut filter = ExtWhenFilter::new( + Some( + Parser::new() + .parse(&json!({"$and": [false, "@key1"]})) + .unwrap(), + ), + true, + ); + assert!(filter.filter_record(mocked_record).is_none()) } #[rstest] fn ready_if_condition_true(mocked_record: Box) { - let mut filter = ExtWhenFilter::new(Some( - Parser::new() - .parse(&json!({"$and": [true, "@key1"]})) - .unwrap(), - )); - let status = ProcessStatus::Ready(Ok(mocked_record)); - assert_matches!(filter.filter_record(status, true), ProcessStatus::Ready(_)) + let mut filter = ExtWhenFilter::new( + Some( + Parser::new() + .parse(&json!({"$and": [true, "@key1"]})) + .unwrap(), + ), + true, + ); + assert!(filter.filter_record(mocked_record).unwrap().is_ok()) } #[rstest] fn ready_with_error_strict(mocked_record: Box) { - let mut filter = ExtWhenFilter::new(Some( - Parser::new() - .parse(&json!({"$and": [true, "@not-exit"]})) - .unwrap(), - )); - let status = ProcessStatus::Ready(Ok(mocked_record)); - assert_matches!( - filter.filter_record(status, true), - ProcessStatus::Ready(Err(_)) - ) + let mut filter = ExtWhenFilter::new( + Some( + Parser::new() + .parse(&json!({"$and": [true, "@not-exit"]})) + .unwrap(), + ), + true, + ); + assert!(filter.filter_record(mocked_record).unwrap().is_err()) } #[rstest] fn ready_without_error(mocked_record: Box) { - let mut filter = ExtWhenFilter::new(Some( - Parser::new() - .parse(&json!({"$and": [true, "@not-exit"]})) - .unwrap(), - )); - let status = ProcessStatus::Ready(Ok(mocked_record)); - assert_matches!(filter.filter_record(status, false), ProcessStatus::Ready(_)) + let mut filter = ExtWhenFilter::new( + Some( + Parser::new() + .parse(&json!({"$and": [true, "@not-exit"]})) + .unwrap(), + ), + false, + ); + assert!( + filter.filter_record(mocked_record).is_none(), + "ignore bad condition" + ) } #[rstest] fn conflict(mut mocked_record: Box) { - let mut filter = ExtWhenFilter::new(Some( - Parser::new() - .parse(&json!({"$and": [true, "@key1"]})) - .unwrap(), - )); - - mocked_record - .labels_mut() - .insert("key1".to_string(), "value1".to_string()); // conflicts with computed key1 - let status = ProcessStatus::Ready(Ok(mocked_record)); - let ProcessStatus::Ready(result) = filter.filter_record(status, true) else { - panic!("Expected ProcessStatus::Ready"); - }; + let mut filter = ExtWhenFilter::new( + Some( + Parser::new() + .parse(&json!({"$and": [true, "@key1"]})) + .unwrap(), + ), + true, + ); + + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key1".to_string(), + "value1".to_string(), + )])) + .computed_labels(mocked_record.meta().computed_labels().clone()) + .build(); + + *mocked_record.meta_mut() = meta; assert_eq!( - result.err().unwrap(), + filter.filter_record(mocked_record).unwrap().err().unwrap(), conflict!("Computed label '@key1' already exists") ) } diff --git a/reductstore/src/replication.rs b/reductstore/src/replication.rs index cd5d804d0..85179cbab 100644 --- a/reductstore/src/replication.rs +++ b/reductstore/src/replication.rs @@ -47,6 +47,13 @@ impl Transaction { Transaction::UpdateRecord(ts) => ts, } } + + pub fn into_timestamp(self) -> u64 { + match self { + Transaction::WriteRecord(ts) => ts, + Transaction::UpdateRecord(ts) => ts, + } + } } impl TryFrom for Transaction { diff --git a/reductstore/src/replication/remote_bucket/client_wrapper.rs b/reductstore/src/replication/remote_bucket/client_wrapper.rs index ba58e4413..08ebbe692 100644 --- a/reductstore/src/replication/remote_bucket/client_wrapper.rs +++ b/reductstore/src/replication/remote_bucket/client_wrapper.rs @@ -10,7 +10,7 @@ use std::collections::BTreeMap; use crate::replication::remote_bucket::ErrorRecordMap; use crate::storage::entry::RecordReader; use reduct_base::error::{ErrorCode, IntEnum, ReductError}; -use reduct_base::io::{ReadRecord, RecordMeta}; +use reduct_base::io::ReadRecord; use reduct_base::unprocessable_entity; use reqwest::header::{HeaderMap, HeaderValue, CONTENT_LENGTH, CONTENT_TYPE}; use reqwest::{Body, Client, Error, Method, Response}; @@ -103,7 +103,7 @@ impl BucketWrapper { let content_length: u64 = if update_only { 0 } else { - records.iter().map(|r| r.content_length()).sum() + records.iter().map(|r| r.meta().content_length()).sum() }; headers.insert( @@ -116,18 +116,19 @@ impl BucketWrapper { ); for record in records { + let meta = record.meta(); let mut header_values = Vec::new(); if update_only { header_values.push("0".to_string()); header_values.push("".to_string()); } else { - header_values.push(record.content_length().to_string()); - header_values.push(record.content_type().to_string()); + header_values.push(meta.content_length().to_string()); + header_values.push(meta.content_type().to_string()); } - if !record.labels().is_empty() { + if !meta.labels().is_empty() { let mut label_headers = vec![]; - for (name, value) in record.labels() { + for (name, value) in meta.labels() { if value.contains(',') { label_headers.push(format!("{}=\"{}\"", name, value)); } else { @@ -140,7 +141,7 @@ impl BucketWrapper { } headers.insert( - HeaderName::from_str(&format!("x-reduct-time-{}", record.timestamp())).unwrap(), + HeaderName::from_str(&format!("x-reduct-time-{}", meta.timestamp())).unwrap(), HeaderValue::from_str(&header_values.join(",").to_string()).unwrap(), ); } @@ -205,7 +206,7 @@ impl BucketWrapper { } fn sort_by_timestamp(records_to_update: &mut Vec) { - records_to_update.sort_by(|a, b| b.timestamp().cmp(&a.timestamp())); + records_to_update.sort_by(|a, b| b.meta().timestamp().cmp(&a.meta().timestamp())); } } diff --git a/reductstore/src/replication/remote_bucket/states/bucket_available.rs b/reductstore/src/replication/remote_bucket/states/bucket_available.rs index 06a009bb8..e225c7bbb 100644 --- a/reductstore/src/replication/remote_bucket/states/bucket_available.rs +++ b/reductstore/src/replication/remote_bucket/states/bucket_available.rs @@ -10,7 +10,7 @@ use crate::storage::entry::RecordReader; use log::{debug, warn}; use reduct_base::error::ErrorCode::MethodNotAllowed; use reduct_base::error::{ErrorCode, ReductError}; -use reduct_base::io::RecordMeta; +use reduct_base::io::ReadRecord; use std::collections::BTreeMap; /// A state when the remote bucket is available. @@ -99,7 +99,7 @@ impl RemoteBucketState for BucketAvailableState { let mut error_map = BTreeMap::new(); for record in &records_to_update { - error_map.insert(record.timestamp(), err.clone()); + error_map.insert(record.meta().timestamp(), err.clone()); } error_map } @@ -110,7 +110,7 @@ impl RemoteBucketState for BucketAvailableState { // Write the records that failed to update with new records. while let Some(record) = records_to_update.pop() { - if error_map.contains_key(&record.timestamp()) { + if error_map.contains_key(&record.meta().timestamp()) { records_to_write.push(record); } } diff --git a/reductstore/src/replication/replication_sender.rs b/reductstore/src/replication/replication_sender.rs index c2fcaadb1..051d4d00a 100644 --- a/reductstore/src/replication/replication_sender.rs +++ b/reductstore/src/replication/replication_sender.rs @@ -99,7 +99,7 @@ impl ReplicationSender { processed_transactions += 1; if let Some(record_to_sync) = record_to_sync { - let record_size = record_to_sync.content_length(); + let record_size = record_to_sync.meta().content_length(); total_size += record_size; batch.push((record_to_sync, transaction)); diff --git a/reductstore/src/replication/transaction_filter.rs b/reductstore/src/replication/transaction_filter.rs index 5999a25d6..61b47be00 100644 --- a/reductstore/src/replication/transaction_filter.rs +++ b/reductstore/src/replication/transaction_filter.rs @@ -4,7 +4,6 @@ use log::warn; use reduct_base::io::RecordMeta; use reduct_base::msg::replication_api::ReplicationSettings; -use reduct_base::Labels; use crate::replication::TransactionNotification; use crate::storage::query::condition::Parser; @@ -19,13 +18,13 @@ pub(super) struct TransactionFilter { query_filters: Vec>, } -impl RecordMeta for TransactionNotification { - fn timestamp(&self) -> u64 { - self.event.timestamp().clone() - } - - fn labels(&self) -> &Labels { - &self.labels +impl Into for TransactionNotification { + fn into(self) -> RecordMeta { + RecordMeta::builder() + .timestamp(self.event.into_timestamp()) + .state(0) + .labels(self.labels) + .build() } } @@ -110,7 +109,8 @@ impl TransactionFilter { // filter out notifications for filter in self.query_filters.iter_mut() { - match filter.filter(notification) { + let meta: RecordMeta = notification.clone().into(); + match filter.filter(&meta) { Ok(false) => return false, Err(err) => { warn!("Error filtering transaction notification: {}", err); @@ -320,9 +320,10 @@ mod tests { #[rstest] fn test_filter_point(notification: TransactionNotification) { - assert_eq!(notification.timestamp(), 0); - assert_eq!(notification.labels(), ¬ification.labels); - assert_eq!(notification.state(), 0); + let meta: RecordMeta = notification.clone().into(); + assert_eq!(meta.timestamp(), 0); + assert_eq!(meta.labels(), ¬ification.labels); + assert_eq!(meta.state(), 0); } } diff --git a/reductstore/src/storage/entry.rs b/reductstore/src/storage/entry.rs index 0e8a1239a..6b4ec0705 100644 --- a/reductstore/src/storage/entry.rs +++ b/reductstore/src/storage/entry.rs @@ -387,7 +387,7 @@ mod tests { use super::*; use reduct_base::error::ErrorCode; - use reduct_base::io::RecordMeta; + use reduct_base::io::ReadRecord; use reduct_base::{no_content, not_found}; use std::thread::sleep; @@ -412,15 +412,15 @@ mod tests { { let reader = rx.blocking_recv().unwrap().unwrap(); - assert_eq!(reader.timestamp(), 1000000); + assert_eq!(reader.meta().timestamp(), 1000000); } { let reader = rx.blocking_recv().unwrap().unwrap(); - assert_eq!(reader.timestamp(), 2000000); + assert_eq!(reader.meta().timestamp(), 2000000); } { let reader = rx.blocking_recv().unwrap().unwrap(); - assert_eq!(reader.timestamp(), 3000000); + assert_eq!(reader.meta().timestamp(), 3000000); } assert_eq!( @@ -454,7 +454,7 @@ mod tests { let rx = entry.get_query_receiver(id).unwrap().upgrade_and_unwrap(); let mut rx = rx.blocking_write(); let reader = rx.blocking_recv().unwrap().unwrap(); - assert_eq!(reader.timestamp(), 1000000); + assert_eq!(reader.meta().timestamp(), 1000000); assert_eq!( rx.blocking_recv().unwrap().err(), Some(no_content!("No content")) @@ -476,7 +476,7 @@ mod tests { Err(e) => panic!("Unexpected error: {:?}", e), } }; - assert_eq!(reader.timestamp(), 2000000); + assert_eq!(reader.meta().timestamp(), 2000000); } sleep(Duration::from_millis(1700)); diff --git a/reductstore/src/storage/entry/entry_loader.rs b/reductstore/src/storage/entry/entry_loader.rs index ad064cd4d..86e5b0785 100644 --- a/reductstore/src/storage/entry/entry_loader.rs +++ b/reductstore/src/storage/entry/entry_loader.rs @@ -356,7 +356,7 @@ mod tests { use super::*; use crate::core::file_cache::FILE_CACHE; - use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::io::ReadRecord; use rstest::{fixture, rstest}; #[rstest] @@ -408,9 +408,9 @@ mod tests { assert_eq!(info.size, 88); let mut rec = entry.begin_read(1).wait().unwrap(); - assert_eq!(rec.timestamp(), 1); - assert_eq!(rec.content_length(), 10); - assert_eq!(rec.content_type(), "text/plain"); + assert_eq!(rec.meta().timestamp(), 1); + assert_eq!(rec.meta().content_length(), 10); + assert_eq!(rec.meta().content_type(), "text/plain"); assert_eq!( rec.blocking_read().unwrap().unwrap(), @@ -418,9 +418,9 @@ mod tests { ); let mut rec = entry.begin_read(2000010).wait().unwrap(); - assert_eq!(rec.timestamp(), 2000010); - assert_eq!(rec.content_length(), 10); - assert_eq!(rec.content_type(), "text/plain"); + assert_eq!(rec.meta().timestamp(), 2000010); + assert_eq!(rec.meta().content_length(), 10); + assert_eq!(rec.meta().content_type(), "text/plain"); assert_eq!( rec.blocking_read().unwrap().unwrap(), diff --git a/reductstore/src/storage/entry/io/record_reader.rs b/reductstore/src/storage/entry/io/record_reader.rs index 8733b1f9a..452615bf4 100644 --- a/reductstore/src/storage/entry/io/record_reader.rs +++ b/reductstore/src/storage/entry/io/record_reader.rs @@ -4,14 +4,14 @@ use crate::core::file_cache::FileWeak; use crate::core::thread_pool::shared_child_isolated; use crate::storage::block_manager::{BlockManager, BlockRef, RecordRx}; -use crate::storage::proto::{ts_to_us, Record}; +use crate::storage::proto::Record; use crate::storage::storage::{CHANNEL_BUFFER_SIZE, IO_OPERATION_TIMEOUT, MAX_IO_BUFFER_SIZE}; use async_trait::async_trait; use bytes::Bytes; use log::error; use reduct_base::error::ReductError; use reduct_base::io::{ReadChunk, ReadRecord, RecordMeta}; -use reduct_base::{internal_server_error, timeout, Labels}; +use reduct_base::{internal_server_error, timeout}; use std::cmp::min; use std::io::Read; use std::io::{Seek, SeekFrom}; @@ -25,13 +25,7 @@ use tokio::sync::mpsc::{channel, Sender}; /// RecordReader is responsible for reading the content of a record from the storage. pub(crate) struct RecordReader { rx: Option, - timestamp: u64, - content_type: String, - length: u64, - last: bool, - labels: Labels, - computed_labels: Labels, - state: i32, + meta: RecordMeta, } struct ReadContext { @@ -129,20 +123,9 @@ impl RecordReader { /// /// * `RecordReader` - The record reader to read the record content in chunks pub fn form_record(record: Record, last: bool) -> Self { - RecordReader { - rx: None, - timestamp: ts_to_us(record.timestamp.as_ref().unwrap()), - length: record.end - record.begin, - content_type: record.content_type.clone(), - labels: record - .labels - .into_iter() - .map(|l| (l.name, l.value)) - .collect(), - computed_labels: Labels::new(), - state: record.state, - last, - } + let mut meta: RecordMeta = record.into(); + meta.set_last(last); + RecordReader { rx: None, meta } } pub fn form_record_with_rx(rx: RecordRx, record: Record, last: bool) -> Self { @@ -152,7 +135,7 @@ impl RecordReader { } pub fn set_last(&mut self, last: bool) { - self.last = last; + self.meta.set_last(last); } fn read(tx: Sender>, ctx: ReadContext) { @@ -217,20 +200,6 @@ impl RecordReader { } } -impl RecordMeta for RecordReader { - fn timestamp(&self) -> u64 { - self.timestamp - } - - fn labels(&self) -> &Labels { - &self.labels - } - - fn state(&self) -> i32 { - self.state - } -} - #[async_trait] impl ReadRecord for RecordReader { async fn read(&mut self) -> ReadChunk { @@ -259,24 +228,8 @@ impl ReadRecord for RecordReader { } } - fn last(&self) -> bool { - self.last - } - - fn computed_labels(&self) -> &Labels { - &self.computed_labels - } - - fn computed_labels_mut(&mut self) -> &mut Labels { - &mut self.computed_labels - } - - fn content_length(&self) -> u64 { - self.length - } - - fn content_type(&self) -> &str { - &self.content_type + fn meta(&self) -> &RecordMeta { + &self.meta } } @@ -383,6 +336,7 @@ mod tests { use crate::core::thread_pool::find_task_group; use crate::storage::entry::tests::get_task_group; + use prost_wkt_types::Timestamp; use std::time::Duration; @@ -446,19 +400,7 @@ mod tests { fn test_state(mut record: Record) { record.state = 1; let reader = RecordReader::form_record(record, false); - assert_eq!(reader.state(), 1); - } - - #[rstest] - fn test_computed_labels(record: Record) { - let mut reader = RecordReader::form_record(record, false); - reader - .computed_labels_mut() - .insert("key".to_string(), "value".to_string()); - assert_eq!( - reader.computed_labels(), - &Labels::from([("key".to_string(), "value".to_string())]) - ); + assert_eq!(reader.meta().state(), 1); } #[rstest] diff --git a/reductstore/src/storage/entry/read_record.rs b/reductstore/src/storage/entry/read_record.rs index cae0ded95..ce26fa038 100644 --- a/reductstore/src/storage/entry/read_record.rs +++ b/reductstore/src/storage/entry/read_record.rs @@ -62,7 +62,7 @@ mod tests { use crate::storage::storage::MAX_IO_BUFFER_SIZE; use bytes::Bytes; use reduct_base::error::ReductError; - use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::io::ReadRecord; use reduct_base::Labels; use rstest::rstest; use std::path::PathBuf; @@ -206,6 +206,6 @@ mod tests { } let reader = entry.begin_read(30 * step).wait().unwrap(); - assert_eq!(reader.timestamp(), 3000000); + assert_eq!(reader.meta().timestamp(), 3000000); } } diff --git a/reductstore/src/storage/entry/remove_records.rs b/reductstore/src/storage/entry/remove_records.rs index 15118293b..21c88bbdb 100644 --- a/reductstore/src/storage/entry/remove_records.rs +++ b/reductstore/src/storage/entry/remove_records.rs @@ -6,7 +6,7 @@ use crate::storage::block_manager::BlockManager; use crate::storage::entry::Entry; use log::warn; use reduct_base::error::{ErrorCode, ReductError}; -use reduct_base::io::RecordMeta; +use reduct_base::io::ReadRecord; use reduct_base::msg::entry_api::QueryEntry; use reduct_base::not_found; use std::collections::BTreeMap; @@ -86,7 +86,7 @@ impl Entry { let result = rx.upgrade()?.blocking_write().blocking_recv(); match result { Some(Ok(rec)) => { - records_to_remove.push(rec.timestamp()); + records_to_remove.push(rec.meta().timestamp()); } Some(Err(ReductError { status: ErrorCode::NoContent, diff --git a/reductstore/src/storage/entry/update_labels.rs b/reductstore/src/storage/entry/update_labels.rs index d5d0edf08..1e7c270e2 100644 --- a/reductstore/src/storage/entry/update_labels.rs +++ b/reductstore/src/storage/entry/update_labels.rs @@ -151,7 +151,7 @@ mod tests { use crate::storage::entry::tests::{entry, write_record_with_labels}; use crate::storage::entry::EntrySettings; - use reduct_base::io::RecordMeta; + use reduct_base::io::ReadRecord; use rstest::rstest; #[rstest] @@ -200,13 +200,13 @@ mod tests { assert_eq!(updated_labels, &expected_labels_3); // check if the records were updated - let labels = entry.begin_read(1).wait().unwrap().labels().clone(); + let labels = entry.begin_read(1).wait().unwrap().meta().labels().clone(); assert_eq!(labels, expected_labels_1); - let labels = entry.begin_read(2).wait().unwrap().labels().clone(); + let labels = entry.begin_read(2).wait().unwrap().meta().labels().clone(); assert_eq!(labels, expected_labels_2); - let labels = entry.begin_read(3).wait().unwrap().labels().clone(); + let labels = entry.begin_read(3).wait().unwrap().meta().labels().clone(); assert_eq!(labels, expected_labels_3); } diff --git a/reductstore/src/storage/proto.rs b/reductstore/src/storage/proto.rs index 6239a778f..dbc63d42b 100644 --- a/reductstore/src/storage/proto.rs +++ b/reductstore/src/storage/proto.rs @@ -2,6 +2,9 @@ // Licensed under the Business Source License 1.1 use prost_wkt_types::Timestamp; +use reduct_base::io::RecordMeta; +use reduct_base::Labels; + include!(concat!(env!("OUT_DIR"), "/reduct.proto.storage.rs")); /// Converts a Timestamp to UNIX microseconds. @@ -17,3 +20,19 @@ pub fn us_to_ts(ts: &u64) -> Timestamp { ..Default::default() } } + +impl Into for Record { + fn into(self) -> RecordMeta { + RecordMeta::builder() + .timestamp(ts_to_us(self.timestamp.as_ref().unwrap())) + .content_length(self.end - self.begin) + .content_type(self.content_type) + .state(self.state) + .labels(Labels::from_iter( + self.labels + .into_iter() + .map(|label| (label.name, label.value)), + )) + .build() + } +} diff --git a/reductstore/src/storage/query.rs b/reductstore/src/storage/query.rs index 8b4c22010..defdba715 100644 --- a/reductstore/src/storage/query.rs +++ b/reductstore/src/storage/query.rs @@ -179,7 +179,7 @@ mod tests { use crate::storage::block_manager::block_index::BlockIndex; use crate::storage::proto::Record; use prost_wkt_types::Timestamp; - use reduct_base::io::RecordMeta; + use reduct_base::io::ReadRecord; use rstest::*; use test_log::test as log_test; use tokio::time::timeout; @@ -241,8 +241,8 @@ mod tests { options, block_manager.clone(), ); - assert_eq!(rx.recv().await.unwrap().unwrap().timestamp(), 0); - assert_eq!(rx.recv().await.unwrap().unwrap().timestamp(), 1); + assert_eq!(rx.recv().await.unwrap().unwrap().meta().timestamp(), 0); + assert_eq!(rx.recv().await.unwrap().unwrap().meta().timestamp(), 1); assert_eq!(rx.recv().await.unwrap().err().unwrap().status, NoContent); assert_eq!(timeout(Duration::from_millis(1000), handle).await, Ok(())); } @@ -263,8 +263,8 @@ mod tests { options, block_manager.clone(), ); - assert_eq!(rx.recv().await.unwrap().unwrap().timestamp(), 0); - assert_eq!(rx.recv().await.unwrap().unwrap().timestamp(), 1); + assert_eq!(rx.recv().await.unwrap().unwrap().meta().timestamp(), 0); + assert_eq!(rx.recv().await.unwrap().unwrap().meta().timestamp(), 1); assert_eq!(rx.recv().await.unwrap().err().unwrap().status, NoContent); block_manager @@ -286,7 +286,7 @@ mod tests { content_type: "".to_string(), }); - assert_eq!(rx.recv().await.unwrap().unwrap().timestamp(), 2); + assert_eq!(rx.recv().await.unwrap().unwrap().meta().timestamp(), 2); assert_eq!(rx.recv().await.unwrap().err().unwrap().status, NoContent); assert!( timeout(Duration::from_millis(1000), handle).await.is_err(), diff --git a/reductstore/src/storage/query/continuous.rs b/reductstore/src/storage/query/continuous.rs index 8bd16af2f..8c7b99c43 100644 --- a/reductstore/src/storage/query/continuous.rs +++ b/reductstore/src/storage/query/continuous.rs @@ -2,12 +2,11 @@ // Licensed under the Business Source License 1.1 use crate::storage::block_manager::BlockManager; +use crate::storage::entry::RecordReader; use crate::storage::query::base::{Query, QueryOptions}; use crate::storage::query::historical::HistoricalQuery; use reduct_base::error::{ErrorCode, ReductError}; - -use crate::storage::entry::RecordReader; -use reduct_base::io::RecordMeta; +use reduct_base::io::ReadRecord; use std::sync::{Arc, RwLock}; pub struct ContinuousQuery { @@ -38,7 +37,7 @@ impl Query for ContinuousQuery { ) -> Result { match self.query.next(block_manager) { Ok(reader) => { - self.next_start = reader.timestamp() + 1; + self.next_start = reader.meta().timestamp() + 1; self.count += 1; Ok(reader) } @@ -80,7 +79,7 @@ mod tests { .unwrap(); { let reader = query.next(block_manager.clone()).unwrap(); - assert_eq!(reader.timestamp(), 1000); + assert_eq!(reader.meta().timestamp(), 1000); } assert_eq!( query.next(block_manager.clone()).err(), diff --git a/reductstore/src/storage/query/filters.rs b/reductstore/src/storage/query/filters.rs index a242bb94a..053bff966 100644 --- a/reductstore/src/storage/query/filters.rs +++ b/reductstore/src/storage/query/filters.rs @@ -22,7 +22,7 @@ pub trait RecordFilter { /// * `Ok(true)` if the record passes the filter, `Ok(false)` otherwise. /// * Err(`ReductError::Interrupt`) if the filter is interrupted /// * `Err(ReductError)` if an error occurs during filtering. - fn filter(&mut self, record: &dyn RecordMeta) -> Result; + fn filter(&mut self, record: &RecordMeta) -> Result; } pub(crate) use each_n::EachNFilter; @@ -34,50 +34,3 @@ use reduct_base::error::ReductError; use reduct_base::io::RecordMeta; pub(crate) use time_range::TimeRangeFilter; pub(crate) use when::WhenFilter; - -#[cfg(test)] -mod tests { - use crate::storage::proto::{ts_to_us, Record}; - use prost_wkt_types::Timestamp; - use reduct_base::io::RecordMeta; - use reduct_base::Labels; - - // Wrapper for Record to implement RecordMeta - // and use it in filter tests - pub(super) struct RecordWrapper { - timestamp: u64, - labels: Labels, - state: i32, - } - - impl RecordMeta for RecordWrapper { - fn timestamp(&self) -> u64 { - self.timestamp - } - - fn labels(&self) -> &Labels { - &self.labels - } - - fn state(&self) -> i32 { - self.state - } - } - - impl From for RecordWrapper { - fn from(record: Record) -> Self { - RecordWrapper { - timestamp: ts_to_us(record.timestamp.as_ref().unwrap_or(&Timestamp { - seconds: 0, - nanos: 0, - })), - labels: record - .labels - .iter() - .map(|l| (l.name.clone(), l.value.clone())) - .collect(), - state: record.state, - } - } - } -} diff --git a/reductstore/src/storage/query/filters/each_n.rs b/reductstore/src/storage/query/filters/each_n.rs index 5fde16944..5a7d6941e 100644 --- a/reductstore/src/storage/query/filters/each_n.rs +++ b/reductstore/src/storage/query/filters/each_n.rs @@ -21,7 +21,7 @@ impl EachNFilter { } impl RecordFilter for EachNFilter { - fn filter(&mut self, _record: &dyn RecordMeta) -> Result { + fn filter(&mut self, _record: &RecordMeta) -> Result { let ret = self.count % self.n == 0; self.count += 1; Ok(ret) @@ -31,24 +31,22 @@ impl RecordFilter for EachNFilter { #[cfg(test)] mod tests { use super::*; - use crate::storage::proto::Record; - use crate::storage::query::filters::tests::RecordWrapper; + use rstest::*; #[rstest] fn test_each_n_filter() { let mut filter = EachNFilter::new(2); - let record = Record::default(); + let meta = RecordMeta::builder().build(); - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap(), "First time should pass"); + assert!(filter.filter(&meta).unwrap(), "First time should pass"); assert!( - !filter.filter(&wrapper).unwrap(), + !filter.filter(&meta).unwrap(), "Second time should not pass" ); - assert!(filter.filter(&wrapper).unwrap(), "Third time should pass"); + assert!(filter.filter(&meta).unwrap(), "Third time should pass"); assert!( - !filter.filter(&wrapper).unwrap(), + !filter.filter(&meta).unwrap(), "Fourth time should not pass" ); } diff --git a/reductstore/src/storage/query/filters/each_s.rs b/reductstore/src/storage/query/filters/each_s.rs index 625d2fd0e..a08c6a873 100644 --- a/reductstore/src/storage/query/filters/each_s.rs +++ b/reductstore/src/storage/query/filters/each_s.rs @@ -24,7 +24,7 @@ impl EachSecondFilter { } impl RecordFilter for EachSecondFilter { - fn filter(&mut self, record: &dyn RecordMeta) -> Result { + fn filter(&mut self, record: &RecordMeta) -> Result { let ret = record.timestamp() as i64 - self.last_time >= (self.s * 1_000_000.0) as i64; if ret { self.last_time = record.timestamp().clone() as i64; @@ -37,39 +37,22 @@ impl RecordFilter for EachSecondFilter { #[cfg(test)] mod tests { use super::*; - use crate::storage::proto::Record; - use crate::storage::query::filters::tests::RecordWrapper; - use prost_wkt_types::Timestamp; + use rstest::*; #[rstest] fn test_each_s_filter() { let mut filter = EachSecondFilter::new(2.0); - let mut record = Record::default(); - record.timestamp = Some(Timestamp { - seconds: 1, - nanos: 0, - }); - - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap()); - assert!(!filter.filter(&wrapper).unwrap()); - - record.timestamp = Some(Timestamp { - seconds: 2, - nanos: 0, - }); + let meta = RecordMeta::builder().timestamp(1000_000).build(); - let wrapper = RecordWrapper::from(record.clone()); - assert!(!filter.filter(&wrapper).unwrap()); + assert!(filter.filter(&meta).unwrap()); + assert!(!filter.filter(&meta).unwrap()); - record.timestamp = Some(Timestamp { - seconds: 3, - nanos: 0, - }); + let meta = RecordMeta::builder().timestamp(2000_000).build(); + assert!(!filter.filter(&meta).unwrap()); - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap()); - assert!(!filter.filter(&wrapper).unwrap()); + let meta = RecordMeta::builder().timestamp(3000_000).build(); + assert!(filter.filter(&meta).unwrap()); + assert!(!filter.filter(&meta).unwrap()); } } diff --git a/reductstore/src/storage/query/filters/exclude.rs b/reductstore/src/storage/query/filters/exclude.rs index f57a70383..05840662d 100644 --- a/reductstore/src/storage/query/filters/exclude.rs +++ b/reductstore/src/storage/query/filters/exclude.rs @@ -28,7 +28,7 @@ impl ExcludeLabelFilter { } impl RecordFilter for ExcludeLabelFilter { - fn filter(&mut self, record: &dyn RecordMeta) -> Result { + fn filter(&mut self, record: &RecordMeta) -> Result { let result = !self .labels .iter() @@ -41,10 +41,6 @@ impl RecordFilter for ExcludeLabelFilter { #[cfg(test)] mod tests { use super::*; - use crate::storage::proto::record::Label; - use crate::storage::proto::Record; - - use crate::storage::query::filters::tests::RecordWrapper; use rstest::*; @@ -54,16 +50,14 @@ mod tests { "key".to_string(), "value".to_string(), )])); - let record = Record { - labels: vec![Label { - name: "key".to_string(), - value: "value".to_string(), - }], - ..Default::default() - }; - let wrapper = RecordWrapper::from(record); - assert!(!filter.filter(&wrapper).unwrap(), "Record should not pass"); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key".to_string(), + "value".to_string(), + )])) + .build(); + assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); } #[rstest] @@ -72,16 +66,14 @@ mod tests { "key".to_string(), "value".to_string(), )])); - let record = Record { - labels: vec![Label { - name: "key".to_string(), - value: "other".to_string(), - }], - ..Default::default() - }; - let wrapper = RecordWrapper::from(record); - assert!(filter.filter(&wrapper).unwrap(), "Record should pass"); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key".to_string(), + "other".to_string(), + )])) + .build(); + assert!(filter.filter(&meta).unwrap(), "Record should pass"); } #[rstest] @@ -90,41 +82,27 @@ mod tests { ("key1".to_string(), "value1".to_string()), ("key2".to_string(), "value2".to_string()), ])); - let record = Record { - labels: vec![ - Label { - name: "key1".to_string(), - value: "value1".to_string(), - }, - Label { - name: "key2".to_string(), - value: "value2".to_string(), - }, - Label { - name: "key3".to_string(), - value: "value3".to_string(), - }, - ], - ..Default::default() - }; - let wrapper = RecordWrapper::from(record); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![ + ("key1".to_string(), "value1".to_string()), + ("key2".to_string(), "value2".to_string()), + ("key3".to_string(), "value3".to_string()), + ])) + .build(); assert!( - !filter.filter(&wrapper).unwrap(), + !filter.filter(&meta).unwrap(), "Record should not pass because it has key1=value1 and key2=value2" ); - let record = Record { - labels: vec![Label { - name: "key1".to_string(), - value: "value1".to_string(), - }], - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key1".to_string(), + "value1".to_string(), + )])) + .build(); assert!( - filter.filter(&wrapper).unwrap(), + filter.filter(&meta).unwrap(), "Record should pass because it has only key1=value1" ); } diff --git a/reductstore/src/storage/query/filters/include.rs b/reductstore/src/storage/query/filters/include.rs index f2085ff34..93e8c9bf3 100644 --- a/reductstore/src/storage/query/filters/include.rs +++ b/reductstore/src/storage/query/filters/include.rs @@ -27,7 +27,7 @@ impl IncludeLabelFilter { } impl RecordFilter for IncludeLabelFilter { - fn filter(&mut self, record: &dyn RecordMeta) -> Result { + fn filter(&mut self, record: &RecordMeta) -> Result { let result = self .labels .iter() @@ -40,9 +40,7 @@ impl RecordFilter for IncludeLabelFilter { #[cfg(test)] mod tests { use super::*; - use crate::storage::proto::record::Label; - use crate::storage::proto::Record; - use crate::storage::query::filters::tests::RecordWrapper; + use rstest::*; #[rstest] @@ -51,16 +49,14 @@ mod tests { "key".to_string(), "value".to_string(), )])); - let record = Record { - labels: vec![Label { - name: "key".to_string(), - value: "value".to_string(), - }], - ..Default::default() - }; - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap(), "Record should pass"); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key".to_string(), + "value".to_string(), + )])) + .build(); + assert!(filter.filter(&meta).unwrap(), "Record should pass"); } #[rstest] @@ -69,16 +65,13 @@ mod tests { "key".to_string(), "value".to_string(), )])); - let record = Record { - labels: vec![Label { - name: "key".to_string(), - value: "other".to_string(), - }], - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record); - assert!(!filter.filter(&wrapper).unwrap(), "Record should not pass"); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key".to_string(), + "other".to_string(), + )])) + .build(); + assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); } #[rstest] @@ -87,41 +80,27 @@ mod tests { ("key1".to_string(), "value1".to_string()), ("key2".to_string(), "value2".to_string()), ])); - let record = Record { - labels: vec![ - Label { - name: "key1".to_string(), - value: "value1".to_string(), - }, - Label { - name: "key2".to_string(), - value: "value2".to_string(), - }, - Label { - name: "key3".to_string(), - value: "value3".to_string(), - }, - ], - ..Default::default() - }; - let wrapper = RecordWrapper::from(record); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![ + ("key1".to_string(), "value1".to_string()), + ("key2".to_string(), "value2".to_string()), + ("key3".to_string(), "value3".to_string()), + ])) + .build(); assert!( - filter.filter(&wrapper).unwrap(), + filter.filter(&meta).unwrap(), "Record should pass because it has key1=value1 and key2=value2" ); - let record = Record { - labels: vec![Label { - name: "key1".to_string(), - value: "value1".to_string(), - }], - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record); + let meta = RecordMeta::builder() + .labels(Labels::from_iter(vec![( + "key1".to_string(), + "value1".to_string(), + )])) + .build(); assert!( - !filter.filter(&wrapper).unwrap(), + !filter.filter(&meta).unwrap(), "Record should not pass because it has only key1=value1" ); } diff --git a/reductstore/src/storage/query/filters/record_state.rs b/reductstore/src/storage/query/filters/record_state.rs index cc2881bb0..2b933568c 100644 --- a/reductstore/src/storage/query/filters/record_state.rs +++ b/reductstore/src/storage/query/filters/record_state.rs @@ -27,7 +27,7 @@ impl RecordStateFilter { } impl RecordFilter for RecordStateFilter { - fn filter(&mut self, record: &dyn RecordMeta) -> Result { + fn filter(&mut self, record: &RecordMeta) -> Result { let result = record.state() == self.state as i32; Ok(result) } @@ -37,31 +37,21 @@ impl RecordFilter for RecordStateFilter { mod tests { use super::*; use crate::storage::proto::record::State; - use crate::storage::proto::Record; - use crate::storage::query::filters::tests::RecordWrapper; + use rstest::*; #[rstest] fn test_record_state_filter() { let mut filter = RecordStateFilter::new(State::Finished); - let record = Record { - state: State::Finished as i32, - ..Default::default() - }; - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap(), "Record should pass"); + let meta = RecordMeta::builder().state(State::Finished as i32).build(); + assert!(filter.filter(&meta).unwrap(), "Record should pass"); } #[rstest] fn test_record_state_filter_no_records() { let mut filter = RecordStateFilter::new(State::Finished); - let record = Record { - state: State::Started as i32, - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record.clone()); - assert!(!filter.filter(&wrapper).unwrap(), "Record should not pass"); + let meta = RecordMeta::builder().state(State::Started as i32).build(); + assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); } } diff --git a/reductstore/src/storage/query/filters/time_range.rs b/reductstore/src/storage/query/filters/time_range.rs index 2aa14f901..7058c166a 100644 --- a/reductstore/src/storage/query/filters/time_range.rs +++ b/reductstore/src/storage/query/filters/time_range.rs @@ -27,7 +27,7 @@ impl TimeRangeFilter { } impl RecordFilter for TimeRangeFilter { - fn filter(&mut self, record: &dyn RecordMeta) -> Result { + fn filter(&mut self, record: &RecordMeta) -> Result { let ts = record.timestamp() as u64; let ret = ts >= self.start && ts < self.stop; if ret { @@ -42,22 +42,16 @@ impl RecordFilter for TimeRangeFilter { #[cfg(test)] mod tests { use super::*; - use crate::storage::proto::{us_to_ts, Record}; - use crate::storage::query::filters::tests::RecordWrapper; + use rstest::*; #[rstest] fn test_time_range_filter() { let mut filter = TimeRangeFilter::new(0, 10); - let record = Record { - timestamp: Some(us_to_ts(&5)), - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap(), "First time should pass"); + let meta = RecordMeta::builder().timestamp(5).build(); + assert!(filter.filter(&meta).unwrap(), "First time should pass"); assert!( - !filter.filter(&wrapper).unwrap(), + !filter.filter(&meta).unwrap(), "Second time should not pass, as we have already returned the record" ); } @@ -65,36 +59,23 @@ mod tests { #[rstest] fn test_time_range_filter_no_records() { let mut filter = TimeRangeFilter::new(0, 10); - let record = Record { - timestamp: Some(us_to_ts(&15)), - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record.clone()); - assert!(!filter.filter(&wrapper).unwrap(), "Record should not pass"); + let meta = RecordMeta::builder().timestamp(15).build(); + assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); } #[rstest] fn test_time_include_start() { let mut filter = TimeRangeFilter::new(0, 10); - let record = Record { - timestamp: Some(us_to_ts(&0)), - ..Default::default() - }; - let wrapper = RecordWrapper::from(record.clone()); - assert!(filter.filter(&wrapper).unwrap(), "Record should pass"); + let meta = RecordMeta::builder().timestamp(0).build(); + assert!(filter.filter(&meta).unwrap(), "Record should pass"); } #[rstest] fn test_time_exclude_end() { let mut filter = TimeRangeFilter::new(0, 10); - let record = Record { - timestamp: Some(us_to_ts(&10)), - ..Default::default() - }; - let wrapper = RecordWrapper::from(record.clone()); - assert!(!filter.filter(&wrapper).unwrap(), "Record should not pass"); + let meta = RecordMeta::builder().timestamp(10).build(); + assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); } } diff --git a/reductstore/src/storage/query/filters/when.rs b/reductstore/src/storage/query/filters/when.rs index 6a016cc95..d04540620 100644 --- a/reductstore/src/storage/query/filters/when.rs +++ b/reductstore/src/storage/query/filters/when.rs @@ -17,7 +17,7 @@ impl WhenFilter { } impl RecordFilter for WhenFilter { - fn filter(&mut self, record: &dyn RecordMeta) -> Result { + fn filter(&mut self, record: &RecordMeta) -> Result { let context = Context::new( record.timestamp(), record @@ -34,10 +34,9 @@ impl RecordFilter for WhenFilter { #[cfg(test)] mod tests { use super::*; - use crate::storage::proto::record::Label; - use crate::storage::proto::Record; + use crate::storage::query::condition::Parser; - use crate::storage::query::filters::tests::RecordWrapper; + use reduct_base::Labels; use rstest::rstest; #[rstest] @@ -47,16 +46,15 @@ mod tests { let condition = parser.parse(&json).unwrap(); let mut filter = WhenFilter::new(condition); - let record = Record { - labels: vec![Label { - name: "label".to_string(), - value: "true".to_string(), - }], - ..Default::default() - }; - - let wrapper = RecordWrapper::from(record.clone()); - let result = filter.filter(&wrapper).unwrap(); + + let meta = RecordMeta::builder() + .timestamp(0) + .labels(Labels::from_iter(vec![( + "label".to_string(), + "true".to_string(), + )])) + .build(); + let result = filter.filter(&meta).unwrap(); assert_eq!(result, true); } } diff --git a/reductstore/src/storage/query/historical.rs b/reductstore/src/storage/query/historical.rs index 6f42ca76a..8430a5ef1 100644 --- a/reductstore/src/storage/query/historical.rs +++ b/reductstore/src/storage/query/historical.rs @@ -14,8 +14,6 @@ use crate::storage::query::filters::{ RecordStateFilter, TimeRangeFilter, WhenFilter, }; use reduct_base::error::{ErrorCode, ReductError}; -use reduct_base::io::RecordMeta; -use reduct_base::Labels; pub struct HistoricalQuery { /// The start time of the query. @@ -147,51 +145,14 @@ impl Query for HistoricalQuery { } } -// Wrapper for RecordMeta to implement RecordMeta for Record -// This is needed because we need different layout for labels in RecordMeta and Record -struct RecordMetaWrapper { - time: u64, - labels: Labels, - state: i32, -} - -impl RecordMeta for RecordMetaWrapper { - fn timestamp(&self) -> u64 { - self.time - } - - fn labels(&self) -> &Labels { - &self.labels - } - - fn state(&self) -> i32 { - self.state - } -} - -impl From for RecordMetaWrapper { - fn from(record: Record) -> Self { - RecordMetaWrapper { - time: ts_to_us(record.timestamp.as_ref().unwrap()), - labels: record - .labels - .iter() - .map(|label| (label.name.clone(), label.value.clone())) - .collect(), - state: record.state, - } - } -} - impl HistoricalQuery { fn filter_records_from_current_block(&mut self) -> Result<(Vec, bool), ReductError> { let block = self.current_block.as_ref().unwrap().read()?; let mut filtered_records = Vec::new(); for record in block.record_index().values() { - let wrapper = RecordMetaWrapper::from(record.clone()); let mut include_record = true; for filter in self.filters.iter_mut() { - match filter.filter(&wrapper) { + match filter.filter(&record.clone().into()) { Ok(false) => { include_record = false; break; @@ -246,7 +207,7 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 1); - assert_eq!(records[0].0.timestamp(), 0); + assert_eq!(records[0].0.meta().timestamp(), 0); assert_eq!(records[0].1, "0123456789"); } @@ -256,9 +217,9 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 2); - assert_eq!(records[0].0.timestamp(), 0); + assert_eq!(records[0].0.meta().timestamp(), 0); assert_eq!(records[0].1, "0123456789"); - assert_eq!(records[1].0.timestamp(), 5); + assert_eq!(records[1].0.meta().timestamp(), 5); assert_eq!(records[1].1, "0123456789"); } @@ -268,11 +229,11 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 3); - assert_eq!(records[0].0.timestamp(), 0); + assert_eq!(records[0].0.meta().timestamp(), 0); assert_eq!(records[0].1, "0123456789"); - assert_eq!(records[1].0.timestamp(), 5); + assert_eq!(records[1].0.meta().timestamp(), 5); assert_eq!(records[1].1, "0123456789"); - assert_eq!(records[2].0.timestamp(), 1000); + assert_eq!(records[2].0.meta().timestamp(), 1000); assert_eq!(records[2].1, "0123456789"); } @@ -293,9 +254,9 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 1); - assert_eq!(records[0].0.timestamp(), 1000); + assert_eq!(records[0].0.meta().timestamp(), 1000); assert_eq!( - records[0].0.labels(), + records[0].0.meta().labels(), &HashMap::from([ ("block".to_string(), "2".to_string()), ("record".to_string(), "1".to_string()), @@ -322,9 +283,9 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 2); - assert_eq!(records[0].0.timestamp(), 5); + assert_eq!(records[0].0.meta().timestamp(), 5); assert_eq!( - records[0].0.labels(), + records[0].0.meta().labels(), &HashMap::from([ ("block".to_string(), "1".to_string()), ("record".to_string(), "2".to_string()), @@ -372,8 +333,8 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 2); - assert_eq!(records[0].0.timestamp(), 0); - assert_eq!(records[1].0.timestamp(), 1000); + assert_eq!(records[0].0.meta().timestamp(), 0); + assert_eq!(records[1].0.meta().timestamp(), 1000); } #[rstest] @@ -390,8 +351,8 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 2); - assert_eq!(records[0].0.timestamp(), 0); - assert_eq!(records[1].0.timestamp(), 1000); + assert_eq!(records[0].0.meta().timestamp(), 0); + assert_eq!(records[1].0.meta().timestamp(), 1000); } #[rstest] @@ -408,7 +369,7 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 1); - assert_eq!(records[0].0.timestamp(), 0); + assert_eq!(records[0].0.meta().timestamp(), 0); } #[rstest] @@ -445,7 +406,7 @@ mod tests { let records = read_to_vector(&mut query, block_manager); assert_eq!(records.len(), 1); - assert_eq!(records[0].0.timestamp(), 0); + assert_eq!(records[0].0.meta().timestamp(), 0); } #[rstest] diff --git a/reductstore/src/storage/query/limited.rs b/reductstore/src/storage/query/limited.rs index f017f325b..cae9349b5 100644 --- a/reductstore/src/storage/query/limited.rs +++ b/reductstore/src/storage/query/limited.rs @@ -52,7 +52,7 @@ mod tests { use super::*; use crate::storage::query::base::tests::block_manager; use reduct_base::error::ErrorCode; - use reduct_base::io::{ReadRecord, RecordMeta}; + use reduct_base::io::ReadRecord; use rstest::rstest; #[rstest] @@ -68,8 +68,8 @@ mod tests { .unwrap(); let reader = query.next(block_manager.clone()).unwrap(); - assert_eq!(reader.timestamp(), 0); - assert!(reader.last()); + assert_eq!(reader.meta().timestamp(), 0); + assert!(reader.meta().last()); assert_eq!( query.next(block_manager).err(), From 6abc691ba9809f3b4f46a3646127f8232f67221b Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 23 May 2025 21:33:23 +0200 Subject: [PATCH 16/93] fix bached test --- reductstore/src/api/entry/read_batched.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index e5085d8b5..2f90f5fa9 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -315,11 +315,10 @@ mod tests { .unwrap(); writer.send(Ok(Some(Bytes::from("Hey!!!")))).await.unwrap(); writer.send(Ok(None)).await.unwrap(); + // let threads finish writing + sleep(Duration::from_millis(1)).await; } - // let threads finish writing - sleep(Duration::from_millis(100)).await; - let query_id = query(&path_to_entry_1, components.clone()).await; let query = Query(HashMap::from_iter(vec![( "q".to_string(), From 9a8c49e4ad932d67f1101cc285c841de893951cd Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 23 May 2025 21:58:07 +0200 Subject: [PATCH 17/93] remove sleeps in batched test --- reductstore/src/api/entry/read_batched.rs | 37 +++++++++++------------ 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index 2f90f5fa9..448455a10 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -286,7 +286,6 @@ mod tests { use reduct_base::Labels; use rstest::*; use tempfile::tempdir; - use tokio::time::sleep; #[rstest] #[case("GET", "Hey!!!")] @@ -300,23 +299,23 @@ mod tests { #[case] _body: String, ) { let components = components.await; - let entry = components - .storage - .get_bucket("bucket-1") - .unwrap() - .upgrade_and_unwrap() - .get_entry("entry-1") - .unwrap() - .upgrade_and_unwrap(); - for time in 10..100 { - let mut writer = entry - .begin_write(time, 6, "text/plain".to_string(), HashMap::new()) - .await - .unwrap(); - writer.send(Ok(Some(Bytes::from("Hey!!!")))).await.unwrap(); - writer.send(Ok(None)).await.unwrap(); - // let threads finish writing - sleep(Duration::from_millis(1)).await; + { + let entry = components + .storage + .get_bucket("bucket-1") + .unwrap() + .upgrade_and_unwrap() + .get_entry("entry-1") + .unwrap() + .upgrade_and_unwrap(); + for time in 10..100 { + let mut writer = entry + .begin_write(time, 6, "text/plain".to_string(), HashMap::new()) + .await + .unwrap(); + writer.send(Ok(Some(Bytes::from("Hey!!!")))).await.unwrap(); + writer.send(Ok(None)).await.unwrap(); + } } let query_id = query(&path_to_entry_1, components.clone()).await; @@ -388,8 +387,6 @@ mod tests { let response = read_batched_records!(); let resp_headers = response.headers(); - println!("{:?}", resp_headers); - assert_eq!( resp_headers["x-reduct-error"], format!("Query {} not found and it might have expired. Check TTL in your query request. Default value 60 sec.", query_id) From 2b2ce71b843fc49fdd9968839fd10033c85febb9 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Mon, 26 May 2025 12:29:43 +0200 Subject: [PATCH 18/93] fix CHANGELOG --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88d799099..4c7e3d21f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Fixed - Fix lock of write channel for small chunks, [PR-834](https://github.com/reductstore/reductstore/pull/834) ->>>>>>> stable ## [1.15.2] - 2025-05-21 From baa11615d6e011458af097c9f88e3bc96d6ce438 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Mon, 26 May 2025 18:02:06 +0200 Subject: [PATCH 19/93] Fix hanging query request if no extension registered (#830) * update CHANGELOG * fix load extension for arm64 macos * fix finishing query in extension repository bypass --- .github/workflows/ci.yml | 10 ++-- CHANGELOG.md | 5 ++ reductstore/src/api/entry/read_batched.rs | 13 +++-- reductstore/src/ext/ext_repository.rs | 28 ++++++++--- reductstore/src/ext/ext_repository/create.rs | 50 +++++++++++++++++++- reductstore/src/ext/ext_repository/load.rs | 31 ++++++++---- reductstore/src/storage/storage.rs | 2 +- 7 files changed, 112 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f8595b1d..47b99aaaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,6 @@ env: jobs: rust_fmt: - runs-on: ubuntu-latest name: Rust Linter steps: @@ -146,19 +145,20 @@ jobs: - build_binaries strategy: matrix: - os: [ubuntu-24.04, windows-2022, macos-13] + os: [ubuntu-24.04, windows-2022, macos-14] include: - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu - os: windows-2022 target: x86_64-pc-windows-gnu - - os: macos-13 - target: x86_64-apple-darwin + - os: macos-14 + target: aarch64-apple-darwin runs-on: ${{ matrix.os }} timeout-minutes: 5 env: RUST_BACKTRACE: 1 + RUST_LOG: debug steps: - uses: actions/download-artifact@v4 with: @@ -187,7 +187,7 @@ jobs: - rust_fmt env: CARGO_TERM_COLOR: always - RUST_LOG: trace # expand logging + RUST_LOG: debug # expand logging steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c7e3d21f..a8de0160d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) - Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) +### Fixed + +- Fix hanging query request if no extension registered, [PR-830](https://github.com/reductstore/reductstore/pull/830) + ## [1.15.3] - 2025-05-26 ## Fixed - Fix lock of write channel for small chunks, [PR-834](https://github.com/reductstore/reductstore/pull/834) + ## [1.15.2] - 2025-05-21 ### Changed diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index 448455a10..8cba89cf8 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -286,8 +286,10 @@ mod tests { use reduct_base::Labels; use rstest::*; use tempfile::tempdir; + use test_log::test as test_log; + use tokio::time::sleep; - #[rstest] + #[test_log(rstest)] #[case("GET", "Hey!!!")] #[case("HEAD", "")] #[tokio::test] @@ -385,6 +387,7 @@ mod tests { ); } + sleep(Duration::from_millis(200)).await; let response = read_batched_records!(); let resp_headers = response.headers(); assert_eq!( @@ -468,7 +471,7 @@ mod tests { let (tx, rx) = tokio::sync::mpsc::channel(1); let rx = Arc::new(AsyncRwLock::new(rx)); drop(tx); - assert!( + assert_eq!( timeout( Duration::from_secs(1), next_record_reader( @@ -481,7 +484,11 @@ mod tests { ) .await .unwrap() - .is_none(), + .unwrap() + .err() + .unwrap() + .status(), + ErrorCode::NoContent, "should return None if the query is closed" ); } diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index 36ed8d3ba..d8836e27b 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -197,12 +197,19 @@ impl ManageExtensions for ExtRepository { let query = match lock.get_mut(&query_id) { Some(query) => query, None => { - return query_rx + let result = query_rx .write() .await .recv() .await - .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)) + .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)); + + if result.is_none() { + // If no record is available, return a no content error to finish the query. + return Some(Err(no_content!("No content"))); + } + + return result; } }; @@ -460,19 +467,26 @@ pub(super) mod tests { use mockall::predicate; use reduct_base::internal_server_error; + use tokio::sync::mpsc; #[rstest] #[tokio::test] async fn test_empty_query() { let mocked_ext_repo = mocked_ext_repo("test-ext", MockIoExtension::new()); - let (tx, rx) = tokio::sync::mpsc::channel(1); + let (tx, rx) = mpsc::channel(1); drop(tx); let query_rx = Arc::new(AsyncRwLock::new(rx)); - assert!(mocked_ext_repo - .fetch_and_process_record(1, query_rx) - .await - .is_none(),); + assert_eq!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .unwrap() + .err() + .unwrap(), + no_content!("No content"), + "Should return no content error when no records are available" + ); } #[rstest] diff --git a/reductstore/src/ext/ext_repository/create.rs b/reductstore/src/ext/ext_repository/create.rs index 2e8e350ae..22eedd6a1 100644 --- a/reductstore/src/ext/ext_repository/create.rs +++ b/reductstore/src/ext/ext_repository/create.rs @@ -8,9 +8,11 @@ use async_trait::async_trait; use reduct_base::error::ReductError; use reduct_base::ext::{BoxedReadRecord, ExtSettings}; use reduct_base::msg::entry_api::QueryEntry; +use reduct_base::no_content; use std::path::PathBuf; use std::sync::Arc; use tokio::sync::RwLock as AsyncRwLock; + pub fn create_ext_repository( external_path: Option, embedded_extensions: Vec>, @@ -55,15 +57,59 @@ pub fn create_ext_repository( _query_id: u64, query_rx: Arc>, ) -> Option> { - query_rx + let result = query_rx .write() .await .recv() .await - .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)) + .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)); + + if result.is_none() { + // If no record is available, return a no content error to finish the query. + return Some(Err(no_content!(""))); + } + + result } } Ok(Box::new(NoExtRepository)) } } + +#[cfg(test)] +mod tests { + use crate::ext::ext_repository::create_ext_repository; + use reduct_base::error::ErrorCode::NoContent; + use reduct_base::ext::ExtSettings; + use reduct_base::msg::server_api::ServerInfo; + use std::sync::Arc; + use tokio::sync::mpsc; + use tokio::sync::RwLock as AsyncRwLock; + + #[tokio::test] + async fn test_no_content_error_returned() { + // Create the dummy extension repository + let ext_repo = create_ext_repository( + None, + vec![], + ExtSettings::builder() + .server_info(ServerInfo::default()) + .build(), + ) + .unwrap(); + + let (tx, rx) = mpsc::channel(1); + let rx = Arc::new(AsyncRwLock::new(rx)); + drop(tx); // Close the sender to simulate no records being available + + // Call fetch_and_process_record, which should return None + let result = ext_repo.fetch_and_process_record(1, rx.clone()).await; + assert!( + result.is_some(), + "Should return Some if no record is available" + ); + let err = result.unwrap().err().unwrap(); + assert_eq!(err.status(), NoContent); + } +} diff --git a/reductstore/src/ext/ext_repository/load.rs b/reductstore/src/ext/ext_repository/load.rs index 4f452741e..e3d95da53 100644 --- a/reductstore/src/ext/ext_repository/load.rs +++ b/reductstore/src/ext/ext_repository/load.rs @@ -13,6 +13,8 @@ use std::path::PathBuf; use std::sync::Arc; use tokio::sync::RwLock as AsyncRwLock; +const EXTENSION_VERSION: &str = "0.2.3"; + impl ExtRepository { pub(super) fn try_load( paths: Vec, @@ -94,7 +96,7 @@ mod tests { info, IoExtensionInfo::builder() .name("test-ext") - .version("0.1.1") + .version(EXTENSION_VERSION) .build() ); } @@ -128,19 +130,30 @@ mod tests { #[fixture] fn ext_repo(ext_settings: ExtSettings) -> ExtRepository { // This is the path to the build directory of the extension from ext_stub crate - const EXTENSION_VERSION: &str = "0.1.1"; - - if !cfg!(target_arch = "x86_64") { - panic!("Unsupported architecture"); - } let file_name = if cfg!(target_os = "linux") { // This is the path to the build directory of the extension from ext_stub crate - "libtest_ext-x86_64-unknown-linux-gnu.so" + if cfg!(target_arch = "aarch64") { + "libtest_ext-aarch64-unknown-linux-gnu.so" + } else if cfg!(target_arch = "x86_64") { + "libtest_ext-x86_64-unknown-linux-gnu.so" + } else { + panic!("Unsupported architecture") + } } else if cfg!(target_os = "macos") { - "libtest_ext-x86_64-apple-darwin.dylib" + if cfg!(target_arch = "aarch64") { + "libtest_ext-aarch64-apple-darwin.dylib" + } else if cfg!(target_arch = "x86_64") { + "libtest_ext-x86_64-apple-darwin.dylib" + } else { + panic!("Unsupported architecture") + } } else if cfg!(target_os = "windows") { - "libtest_ext-x86_64-pc-windows-gnu.dll" + if cfg!(target_arch = "x86_64") { + "libtest_ext-x86_64-pc-windows-gnu.dll" + } else { + panic!("Unsupported architecture") + } } else { panic!("Unsupported platform") }; diff --git a/reductstore/src/storage/storage.rs b/reductstore/src/storage/storage.rs index bb792ae58..3c2d34a26 100644 --- a/reductstore/src/storage/storage.rs +++ b/reductstore/src/storage/storage.rs @@ -142,7 +142,7 @@ impl Storage { /// /// * `Bucket` - The bucket or an HTTPError pub(crate) fn get_bucket(&self, name: &str) -> Result, ReductError> { - let buckets = self.buckets.read().unwrap(); + let buckets = self.buckets.read()?; match buckets.get(name) { Some(bucket) => Ok(Arc::clone(bucket).into()), None => Err(ReductError::not_found( From 3fc273fe0281ebc461f3c36ac42117cf585da64a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 May 2025 18:02:26 +0200 Subject: [PATCH 20/93] Bump tokio from 1.45.0 to 1.45.1 (#831) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.45.0 to 1.45.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.45.0...tokio-1.45.1) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.45.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reduct_base/Cargo.toml | 2 +- reductstore/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e50d8faac..5acb5b182 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2712,9 +2712,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 27187d768..2ce8bbd75 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -34,7 +34,7 @@ url = "2.5.4" http = "1.2.0" bytes = "1.10.0" async-trait = { version = "0.1.87" , optional = true } -tokio = { version = "1.45.0", optional = true, features = ["default", "rt", "time"] } +tokio = { version = "1.45.1", optional = true, features = ["default", "rt", "time"] } log = "0.4.0" thread-id = "5.0.0" futures = "0.3.31" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 4eb64d72b..af2df99b5 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -42,7 +42,7 @@ regex = "1.11.1" bytes = "1.10.1" axum = { version = "0.8.4", features = ["default", "macros"] } axum-extra = { version = "0.10.0", features = ["default", "typed-header"] } -tokio = { version = "1.45.0", features = ["full"] } +tokio = { version = "1.45.1", features = ["full"] } hyper = { version = "1.6.0", features = ["full"] } futures-util = "0.3.31" axum-server = { version = "0.7.1", features = ["tls-rustls"] } From 323c51b2728ce1b0143f3e5506cbc9f4c6face1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 00:36:18 +0200 Subject: [PATCH 21/93] Bump zip from 3.0.0 to 4.0.0 (#832) Bumps [zip](https://github.com/zip-rs/zip2) from 3.0.0 to 4.0.0. - [Release notes](https://github.com/zip-rs/zip2/releases) - [Changelog](https://github.com/zip-rs/zip2/blob/master/CHANGELOG.md) - [Commits](https://github.com/zip-rs/zip2/compare/v3.0.0...v4.0.0) --- updated-dependencies: - dependency-name: zip dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 72 ++++++++++++++---------------------------- reductstore/Cargo.toml | 2 +- 2 files changed, 24 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5acb5b182..29f784d36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -520,21 +520,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -1408,6 +1393,26 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "liblzma" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libz-rs-sys" version = "0.5.0" @@ -1457,27 +1462,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchers" version = "0.1.0" @@ -3383,15 +3367,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.7.5" @@ -3501,9 +3476,9 @@ dependencies = [ [[package]] name = "zip" -version = "3.0.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" +checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" dependencies = [ "aes", "arbitrary", @@ -3515,12 +3490,11 @@ dependencies = [ "getrandom 0.3.2", "hmac", "indexmap", - "lzma-rs", + "liblzma", "memchr", "pbkdf2", "sha1", "time", - "xz2", "zeroize", "zopfli", "zstd", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index af2df99b5..ddf52e9c8 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -31,7 +31,7 @@ reduct-base = { path = "../reduct_base", version = "1.15.0", features = ["ext"] reduct-macros = { path = "../reduct_macros", version = "1.15.0" } chrono = { version = "0.4.41", features = ["serde"] } -zip = "3.0.0" +zip = "4.0.0" tempfile = "3.20.0" hex = "0.4.3" prost-wkt-types = "0.6.1" From adc97768cff9566103aabb192c983c0d7c8ba8c4 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 27 May 2025 16:39:49 +0200 Subject: [PATCH 22/93] Run all operands after a compute-staged one on the compute stage (#835) * refactoring * run all conditions after a computed on the computed stage * improve error message without dollar * fix staging * update CHANGELOG * Update reductstore/src/storage/query/condition/parser.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + reductstore/src/ext/ext_repository/load.rs | 4 +- .../src/storage/query/condition/parser.rs | 239 ++++++++++++------ 3 files changed, 171 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8de0160d..3f120f8f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) - Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) +- Run all operands after a compute-staged one on the compute stage, [PR-835](https://github.com/reductstore/reductstore/pull/835) ### Fixed diff --git a/reductstore/src/ext/ext_repository/load.rs b/reductstore/src/ext/ext_repository/load.rs index e3d95da53..0ebc9b1e2 100644 --- a/reductstore/src/ext/ext_repository/load.rs +++ b/reductstore/src/ext/ext_repository/load.rs @@ -13,8 +13,6 @@ use std::path::PathBuf; use std::sync::Arc; use tokio::sync::RwLock as AsyncRwLock; -const EXTENSION_VERSION: &str = "0.2.3"; - impl ExtRepository { pub(super) fn try_load( paths: Vec, @@ -83,6 +81,8 @@ mod tests { use tempfile::tempdir; use test_log::test as log_test; + const EXTENSION_VERSION: &str = "0.2.3"; + #[log_test(rstest)] fn test_load_extension(ext_repo: ExtRepository) { assert_eq!(ext_repo.extension_map.len(), 1); diff --git a/reductstore/src/storage/query/condition/parser.rs b/reductstore/src/storage/query/condition/parser.rs index 1457e1273..6555bd87d 100644 --- a/reductstore/src/storage/query/condition/parser.rs +++ b/reductstore/src/storage/query/condition/parser.rs @@ -15,7 +15,7 @@ use crate::storage::query::condition::value::Value; use crate::storage::query::condition::{Boxed, BoxedNode, Context, EvaluationStage, Node}; use reduct_base::error::ReductError; use reduct_base::unprocessable_entity; -use serde_json::{Map, Value as JsonValue}; +use serde_json::{Map, Number, Value as JsonValue}; /// Parses a JSON object into a condition tree. pub(crate) struct Parser {} @@ -23,23 +23,16 @@ pub(crate) struct Parser {} /// A node in a condition tree. /// /// It evaluates the node in the given context on different stages. -struct StagedAllOff { +struct StagedAllOf { operands: Vec, } -impl Node for StagedAllOff { +impl Node for StagedAllOf { fn apply(&mut self, context: &Context) -> Result { - for operand in self.operands.iter_mut() { - // Filter out operands that are not in the current stage - if operand.stage() == &context.stage { - let value = operand.apply(context)?; - if !value.as_bool()? { - return Ok(Value::Bool(false)); - } - } + match context.stage { + EvaluationStage::Retrieve => self.apply_retrieve_stage(context), + EvaluationStage::Compute => self.apply_compute_stage(context), } - - Ok(Value::Bool(true)) } fn operands(&self) -> &Vec { @@ -51,69 +44,69 @@ impl Node for StagedAllOff { } } -impl Boxed for StagedAllOff { +impl Boxed for StagedAllOf { fn boxed(operands: Vec) -> Result { Ok(Box::new(Self { operands })) } } -impl Parser { - pub fn parse(&self, json: &JsonValue) -> Result { - let expressions = self.parse_intern(json)?; - Ok(StagedAllOff::boxed(expressions)?) - } - - fn parse_intern(&self, json: &JsonValue) -> Result, ReductError> { - match json { - JsonValue::Object(map) => { - let mut expressions = vec![]; - for (key, value) in map.iter() { - if let JsonValue::Array(operand_list) = value { - // Parse array notation e.g. {"$and": [true, true]} - expressions.push(self.parse_array_syntax(key, operand_list)?); - } else if let JsonValue::Object(operator_right_operand) = value { - // Parse object notation e.g. {"&label": {"$and": true}} - expressions.push(self.parse_object_syntax(key, operator_right_operand)?); - } else { - // For unary operators, we need to parse the value - let operands = self.parse_intern(value)?; - let operator = Self::parse_operator(key, operands)?; - expressions.push(operator); - } - } +impl StagedAllOf { + fn apply_retrieve_stage(&mut self, context: &Context) -> Result { + // In the retrieve stage, we evaluate all operands before the first compute-staged one + for operand in &mut self.operands { + if operand.stage() == &EvaluationStage::Compute { + break; + } - // We use AND operator to aggregate results from all expressions - Ok(expressions) + let value = operand.apply(context)?; + if !value.as_bool()? { + return Ok(Value::Bool(false)); } + } - JsonValue::Bool(value) => Ok(vec![Constant::boxed(Value::Bool(*value))]), + Ok(Value::Bool(true)) + } - JsonValue::Number(value) => { - if value.is_i64() || value.is_u64() { - Ok(vec![Constant::boxed(Value::Int(value.as_i64().unwrap()))]) - } else { - Ok(vec![Constant::boxed(Value::Float(value.as_f64().unwrap()))]) - } - } - JsonValue::String(value) => { - if value.starts_with("&") { - Ok(vec![Reference::boxed( - value[1..].to_string(), - EvaluationStage::Retrieve, - )]) - } else if value.starts_with("@") { - Ok(vec![Reference::boxed( - value[1..].to_string(), - EvaluationStage::Compute, - )]) - } else if value.starts_with("$") { - // operator without operands (nullary) - Ok(vec![Self::parse_operator(value, vec![])?]) - } else { - Ok(vec![Constant::boxed(Value::String(value.clone()))]) + fn apply_compute_stage(&mut self, context: &Context) -> Result { + // In the compute stage, we evaluate all operands after the first compute-staged one + let mut compute_operand_detected = false; + for operand in &mut self.operands { + if operand.stage() == &EvaluationStage::Compute || compute_operand_detected { + compute_operand_detected = true; + + let value = operand.apply(context)?; + if !value.as_bool()? { + return Ok(Value::Bool(false)); } } + } + Ok(Value::Bool(true)) + } +} + +impl Parser { + /// Parses a JSON object into a condition tree. + /// + /// # Arguments + /// + /// * `json` - A JSON value representing the condition. + /// + /// # Returns + /// + /// A boxed node representing the condition tree. + /// The root node is a `StagedAllOf` that aggregates all expressions + pub fn parse(&self, json: &JsonValue) -> Result { + let expressions = Self::parse_recursively(json)?; + Ok(StagedAllOf::boxed(expressions)?) + } + + fn parse_recursively(json: &JsonValue) -> Result, ReductError> { + match json { + JsonValue::Object(map) => Self::parse_object(map), + JsonValue::Bool(value) => Self::parse_bool(value), + JsonValue::Number(value) => Self::parse_number(value), + JsonValue::String(value) => Self::parse_string(value), JsonValue::Array(_) => Err(unprocessable_entity!( "Array type is not supported: {}", json @@ -125,24 +118,75 @@ impl Parser { } } + fn parse_object(map: &Map) -> Result, ReductError> { + let mut expressions = vec![]; + for (key, value) in map.iter() { + if let JsonValue::Array(operand_list) = value { + // Parse array notation e.g. {"$and": [true, true]} + expressions.push(Self::parse_array_syntax(key, operand_list)?); + } else if let JsonValue::Object(operator_right_operand) = value { + // Parse object notation e.g. {"&label": {"$and": true}} + expressions.push(Self::parse_object_syntax(key, operator_right_operand)?); + } else { + // For unary operators, we need to parse the value + let operands = Self::parse_recursively(value)?; + let operator = Self::parse_operator(key, operands)?; + expressions.push(operator); + } + } + + // We use AND operator to aggregate results from all expressions + Ok(expressions) + } + + fn parse_bool(value: &bool) -> Result, ReductError> { + Ok(vec![Constant::boxed(Value::Bool(*value))]) + } + + fn parse_number(value: &Number) -> Result, ReductError> { + if value.is_i64() || value.is_u64() { + Ok(vec![Constant::boxed(Value::Int(value.as_i64().unwrap()))]) + } else { + Ok(vec![Constant::boxed(Value::Float(value.as_f64().unwrap()))]) + } + } + + fn parse_string(value: &str) -> Result, ReductError> { + if value.starts_with("&") { + Ok(vec![Reference::boxed( + value[1..].to_string(), + EvaluationStage::Retrieve, + )]) + } else if value.starts_with("@") { + Ok(vec![Reference::boxed( + value[1..].to_string(), + EvaluationStage::Compute, + )]) + } else if value.starts_with("$") { + // operator without operands (nullary) + Ok(vec![Self::parse_operator(value, vec![])?]) + } else { + Ok(vec![Constant::boxed(Value::String(value.clone()))]) + } + } + fn parse_array_syntax( - &self, operator: &str, json_operands: &Vec, ) -> Result { let mut operands = vec![]; for operand in json_operands { - operands.extend(self.parse_intern(operand)?); + operands.extend(Self::parse_recursively(operand)?); } Self::parse_operator(operator, operands) } fn parse_object_syntax( - &self, left_operand: &str, op_right_operand: &Map, ) -> Result { - let mut left_operand = self.parse_intern(&JsonValue::String(left_operand.to_string()))?; + let mut left_operand = + Self::parse_recursively(&JsonValue::String(left_operand.to_string()))?; if op_right_operand.len() != 1 { return Err(unprocessable_entity!( "Object notation must have exactly one operator" @@ -150,12 +194,19 @@ impl Parser { } let (operator, operand) = op_right_operand.iter().next().unwrap(); - let right_operand = self.parse_intern(operand)?; + let right_operand = Self::parse_recursively(operand)?; left_operand.extend(right_operand); Self::parse_operator(operator, left_operand) } fn parse_operator(operator: &str, operands: Vec) -> Result { + if !operator.starts_with("$") { + return Err(unprocessable_entity!( + "Operator '{}' must start with '$'", + operator + )); + } + match operator { // Aggregation operators "$each_n" => EachN::boxed(operands), @@ -363,6 +414,18 @@ mod tests { ); } + #[rstest] + fn test_parser_invalid_operator_without_dollar(parser: Parser) { + let json = json!({ + "and": [true, true] + }); + let result = parser.parse(&json); + assert_eq!( + result.err().unwrap().to_string(), + "[UnprocessableEntity] Operator 'and' must start with '$'" + ); + } + mod parse_operators { use super::*; #[rstest] @@ -440,7 +503,7 @@ mod tests { Constant::boxed(Value::Bool(true)), Constant::boxed(Value::Bool(false)), ]; - let staged_all_of = StagedAllOff::boxed(operands); + let staged_all_of = StagedAllOf::boxed(operands); assert_eq!( staged_all_of.unwrap().print(), "AllOf([Bool(true), Bool(false)])" @@ -453,10 +516,44 @@ mod tests { Constant::boxed(Value::Bool(false)), // ignored because not in stage ]; - let mut staged_all_of = StagedAllOff::boxed(operands).unwrap(); + let mut staged_all_of = StagedAllOf::boxed(operands).unwrap(); let context = Context::new(0, HashMap::new(), EvaluationStage::Compute); assert_eq!(staged_all_of.apply(&context).unwrap(), Value::Bool(true)); } + + #[rstest] + fn test_all_condition_after_compute_operand() { + let operands: Vec = vec![ + Reference::boxed("label".to_string(), EvaluationStage::Compute), + Constant::boxed(Value::Bool(false)), + ]; + + let mut staged_all_of = StagedAllOf::boxed(operands).unwrap(); + let context = Context::new( + 0, + HashMap::from_iter(vec![("label", "true")]), + EvaluationStage::Compute, + ); + assert_eq!(staged_all_of.apply(&context).unwrap(), Value::Bool(false), + "Must be false because the last retrieved value is false and it is used on the Compute stage"); + } + + #[rstest] + fn test_all_condition_before_compute_operand() { + let operands: Vec = vec![ + Constant::boxed(Value::Bool(true)), + Reference::boxed("label".to_string(), EvaluationStage::Compute), + ]; + + let mut staged_all_of = StagedAllOf::boxed(operands).unwrap(); + let context = Context::new( + 0, + HashMap::from_iter(vec![("label", "false")]), + EvaluationStage::Retrieve, + ); + assert_eq!(staged_all_of.apply(&context).unwrap(), Value::Bool(true), + "Must be true because the last retrieved value, and compute-staged label is ignored"); + } } #[fixture] From b21de7cabdf858e420f462e773a036a3515ef3b6 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 27 May 2025 16:45:04 +0200 Subject: [PATCH 23/93] fix build --- reductstore/src/storage/query/condition/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reductstore/src/storage/query/condition/parser.rs b/reductstore/src/storage/query/condition/parser.rs index 6555bd87d..baa90b683 100644 --- a/reductstore/src/storage/query/condition/parser.rs +++ b/reductstore/src/storage/query/condition/parser.rs @@ -166,7 +166,7 @@ impl Parser { // operator without operands (nullary) Ok(vec![Self::parse_operator(value, vec![])?]) } else { - Ok(vec![Constant::boxed(Value::String(value.clone()))]) + Ok(vec![Constant::boxed(Value::String(value.to_string()))]) } } From 95d12ac5ffad992cad77409bcc4459b88ab2e437 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 28 May 2025 08:38:15 +0200 Subject: [PATCH 24/93] Update yanked zip dependency (#837) * update yanked zip dependency * update CHANGELOG --- CHANGELOG.md | 4 ++ Cargo.lock | 93 +++++++++++++++++++----------------------- reductstore/Cargo.toml | 2 +- 3 files changed, 46 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e77bfa4..f43f6ed72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## Fixed + +- Update yanked zip dependency to 4.0.0, [PR-837](https://github.com/reductstore/reductstore/pull/837) + ## [1.15.3] - 2025-05-26 ## Fixed diff --git a/Cargo.lock b/Cargo.lock index 5c14cbed3..8c91279d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -520,21 +520,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.4.2" @@ -725,11 +710,12 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -1407,6 +1393,35 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "liblzma" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -1447,27 +1462,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "lzma-rs" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" -dependencies = [ - "byteorder", - "crc", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "matchers" version = "0.1.0" @@ -3372,15 +3366,6 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.7.5" @@ -3490,32 +3475,36 @@ dependencies = [ [[package]] name = "zip" -version = "2.6.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" dependencies = [ "aes", "arbitrary", "bzip2", "constant_time_eq", "crc32fast", - "crossbeam-utils", "deflate64", "flate2", "getrandom 0.3.2", "hmac", "indexmap", - "lzma-rs", + "liblzma", "memchr", "pbkdf2", "sha1", "time", - "xz2", "zeroize", "zopfli", "zstd", ] +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zopfli" version = "0.8.1" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 53203436e..282d12a84 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -30,7 +30,7 @@ reduct-base = { path = "../reduct_base", version = "1.15.0", features = ["ext"] reduct-macros = { path = "../reduct_macros", version = "1.15.0" } chrono = { version = "0.4.41", features = ["serde"] } -zip = "2.6.1" +zip = "4.0.0" tempfile = "3.19.1" hex = "0.4.3" prost = "0.13.1" From 3ad37d16d30194b1d7c2ec65b309ba6f3304afce Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 28 May 2025 08:40:24 +0200 Subject: [PATCH 25/93] release v1.15.4 --- CHANGELOG.md | 10 +++++++--- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f43f6ed72..a0b86cba9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## Fixed +## [1.15.4] - 2025-05-28 + +### Fixed - Update yanked zip dependency to 4.0.0, [PR-837](https://github.com/reductstore/reductstore/pull/837) ## [1.15.3] - 2025-05-26 -## Fixed +### Fixed - Fix lock of write channel for small chunks, [PR-834](https://github.com/reductstore/reductstore/pull/834) @@ -1029,7 +1031,9 @@ reduct-rs: `ReductClient.url`, `ReductClient.token`, `ReductCientBuilder.try_bui - Initial release with basic HTTP API and FIFO bucket quota -[Unreleased]: https://github.com/reductstore/reductstore/compare/v1.15.3...HEAD +[Unreleased]: https://github.com/reductstore/reductstore/compare/v1.15.4...HEAD + +[1.15.4]: https://github.com/reductstore/reductstore/compare/v1.15.3...v1.15.4 [1.15.3]: https://github.com/reductstore/reductstore/compare/v1.15.2...v1.15.3 diff --git a/Cargo.lock b/Cargo.lock index 8c91279d6..bb8c7ab6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1995,7 +1995,7 @@ dependencies = [ [[package]] name = "reduct-base" -version = "1.15.3" +version = "1.15.4" dependencies = [ "async-trait", "bytes", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "reduct-macros" -version = "1.15.3" +version = "1.15.4" dependencies = [ "quote", "syn 2.0.101", @@ -2021,7 +2021,7 @@ dependencies = [ [[package]] name = "reductstore" -version = "1.15.3" +version = "1.15.4" dependencies = [ "assert_matches", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 8a4cfb9dd..a60ac5fb8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.15.3" +version = "1.15.4" authors = ["Alexey Timin ", "ReductSoftware UG "] edition = "2021" rust-version = "1.85.0" From 463f8bf602904460ffda74fe72410eec7b38342a Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 28 May 2025 17:02:39 +0200 Subject: [PATCH 26/93] Replace auto-staging for extension filtering with when condition in ext parameter (#838) * add when condition to ext parameter instead autostaging * remove unused code * update CHANGELOG --- CHANGELOG.md | 1 + reductstore/src/ext/ext_repository.rs | 83 ++++++---- reductstore/src/ext/filter.rs | 46 +----- reductstore/src/storage/query/condition.rs | 33 +--- .../query/condition/computed_reference.rs | 72 +++++++++ .../src/storage/query/condition/constant.rs | 24 +-- .../condition/operators/aggregation/each_n.rs | 4 - .../condition/operators/aggregation/each_t.rs | 4 - .../condition/operators/aggregation/limit.rs | 4 - .../condition/operators/arithmetic/abs.rs | 4 - .../condition/operators/arithmetic/add.rs | 4 - .../condition/operators/arithmetic/div.rs | 4 - .../condition/operators/arithmetic/div_num.rs | 4 - .../condition/operators/arithmetic/mult.rs | 4 - .../condition/operators/arithmetic/rem.rs | 4 - .../condition/operators/arithmetic/sub.rs | 4 - .../condition/operators/comparison/eq.rs | 4 - .../condition/operators/comparison/gt.rs | 4 - .../condition/operators/comparison/gte.rs | 4 - .../condition/operators/comparison/lt.rs | 4 - .../condition/operators/comparison/lte.rs | 4 - .../condition/operators/comparison/ne.rs | 4 - .../condition/operators/logical/all_of.rs | 4 - .../condition/operators/logical/any_of.rs | 4 - .../query/condition/operators/logical/in.rs | 4 - .../query/condition/operators/logical/nin.rs | 3 - .../condition/operators/logical/none_of.rs | 4 - .../condition/operators/logical/one_of.rs | 4 - .../query/condition/operators/misc/cast.rs | 4 - .../query/condition/operators/misc/exists.rs | 4 - .../query/condition/operators/misc/ref.rs | 4 - .../condition/operators/misc/timestamp.rs | 18 +-- .../condition/operators/string/contains.rs | 4 - .../condition/operators/string/ends_with.rs | 4 - .../condition/operators/string/starts_with.rs | 4 - .../src/storage/query/condition/parser.rs | 151 +----------------- .../src/storage/query/condition/reference.rs | 47 +----- reductstore/src/storage/query/filters/when.rs | 8 +- 38 files changed, 166 insertions(+), 428 deletions(-) create mode 100644 reductstore/src/storage/query/condition/computed_reference.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0069d5223..6154c0978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) - Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) - Run all operands after a compute-staged one on the compute stage, [PR-835](https://github.com/reductstore/reductstore/pull/835) +- Replace auto-staging for extension filtering with when condition in ext parameter, [PR-838](https://github.com/reductstore/reductstore/pull/838) ### Fixed diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index d8836e27b..14029c637 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -7,7 +7,7 @@ mod load; use crate::asset::asset_manager::ManageStaticAsset; use crate::ext::filter::ExtWhenFilter; use crate::storage::query::base::QueryOptions; -use crate::storage::query::condition::{EvaluationStage, Parser}; +use crate::storage::query::condition::Parser; use crate::storage::query::QueryRx; use async_trait::async_trait; use dlopen2::wrapper::{Container, WrapperApi}; @@ -104,12 +104,21 @@ impl ManageExtensions for ExtRepository { query_id: u64, bucket_name: &str, entry_name: &str, - query_request: QueryEntry, + mut query_request: QueryEntry, ) -> Result<(), ReductError> { let mut query_map = self.query_map.write().await; - let ext_params = query_request.ext.as_ref(); - let controllers = if ext_params.is_some() && ext_params.unwrap().is_object() { - let ext_query = ext_params.unwrap().as_object().unwrap(); + let ext_params = query_request.ext.as_mut(); + let controllers = if ext_params.is_some() && ext_params.as_ref().unwrap().is_object() { + let ext_query = ext_params.unwrap().as_object_mut().unwrap(); + + // check if the query has references to computed labels and no extension is found + let condition = if let Some(condition) = ext_query.remove("when") { + let node = Parser::new().parse(&condition)?; + Some(node) + } else { + None + }; + if ext_query.iter().count() > 1 { return Err(unprocessable_entity!( "Multiple extensions are not supported in query id={}", @@ -129,7 +138,7 @@ impl ManageExtensions for ExtRepository { ext.write() .await .query(bucket_name, entry_name, &query_request)?; - Some((processor, commiter)) + Some((processor, commiter, condition)) } else { return Err(unprocessable_entity!( "Unknown extension '{}' in query id={}", @@ -155,22 +164,9 @@ impl ManageExtensions for ExtRepository { query_map.remove(&key); } - // check if the query has references to computed labels and no extension is found - let condition = if let Some(condition) = &query_options.when { - let node = Parser::new().parse(condition)?; - if controllers.is_none() && node.stage() == &EvaluationStage::Compute { - return Err(unprocessable_entity!( - "There is at least one reference to computed labels but no extension is found" - )); - } - Some(node) - } else { - None - }; - - let condition_filter = ExtWhenFilter::new(condition, query_options.strict); + if let Some((processor, commiter, condition)) = controllers { + let condition_filter = ExtWhenFilter::new(condition, true); - if let Some((processor, commiter)) = controllers { query_map.insert(query_id, { QueryContext { query: query_options, @@ -297,7 +293,9 @@ pub(super) mod tests { mod register_query { use super::*; + use mockall::predicate::always; + use reduct_base::not_found; use std::time::Duration; #[rstest] @@ -353,24 +351,43 @@ pub(super) mod tests { #[rstest] #[tokio::test] - async fn test_without_ext_but_computed_labels(mut mock_ext: MockIoExtension) { + async fn test_when_parsing( + mut mock_ext: MockIoExtension, + processor: BoxedProcessor, + commiter: BoxedCommiter, + ) { let query = QueryEntry { - when: Some(json!({"@label": { "$eq": "value" }})), + ext: Some(json!({ + "test-ext": {}, + "when": {"@label": {"$eq": "value"}}, + })), ..Default::default() }; - mock_ext.expect_query().never(); + mock_ext + .expect_query() + .with(eq("bucket"), eq("entry"), always()) + .return_once(|_, _, _| Ok((processor, commiter))); - let mocked_ext_repo = mocked_ext_repo("test", mock_ext); + let mocked_ext_repo = mocked_ext_repo("test-ext", mock_ext); + + assert!(mocked_ext_repo + .register_query(1, "bucket", "entry", query) + .await + .is_ok(),); + // make sure we parsed condition correctly + let mut query_map = mocked_ext_repo.query_map.write().await; + assert_eq!(query_map.len(), 1, "Query should be registered"); + let query_context = query_map.get_mut(&1).unwrap(); assert_eq!( - mocked_ext_repo - .register_query(1, "bucket", "entry", query) - .await + query_context + .condition_filter + .filter_record(Box::new(MockRecord::new("not-in-when", "val"))) + .unwrap() .err() .unwrap(), - unprocessable_entity!( - "There is at least one reference to computed labels but no extension is found" - ) + not_found!("Reference '@label' not found"), + "Condition should be parsed and applied" ); } @@ -850,10 +867,6 @@ pub(super) mod tests { MockRecord { meta } } - pub fn meta_mut(&mut self) -> &mut RecordMeta { - &mut self.meta - } - pub fn boxed(key: &str, val: &str) -> BoxedReadRecord { Box::new(MockRecord::new(key, val)) } diff --git a/reductstore/src/ext/filter.rs b/reductstore/src/ext/filter.rs index 92fda9299..0c1346723 100644 --- a/reductstore/src/ext/filter.rs +++ b/reductstore/src/ext/filter.rs @@ -1,8 +1,7 @@ // Copyright 2025 ReductSoftware UG // Licensed under the Business Source License 1.1 -use crate::storage::query::condition::{BoxedNode, Context, EvaluationStage}; -use reduct_base::conflict; +use crate::storage::query::condition::{BoxedNode, Context}; use reduct_base::error::ReductError; use reduct_base::ext::BoxedReadRecord; use std::collections::HashMap; @@ -45,19 +44,19 @@ impl ExtWhenFilter { fn filter_with_computed(&mut self, reader: &BoxedReadRecord) -> Result { let meta = reader.meta(); - let mut labels = meta + let labels = meta .labels() .iter() .map(|(k, v)| (k.as_str(), v.as_str())) .collect::>(); - for (k, v) in meta.computed_labels() { - if labels.insert(k, v).is_some() { - return Err(conflict!("Computed label '@{}' already exists", k)); - } - } + let computed_labels = meta + .computed_labels() + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect::>(); - let context = Context::new(meta.timestamp(), labels, EvaluationStage::Compute); + let context = Context::new(meta.timestamp(), labels, computed_labels); Ok(self .condition .as_mut() @@ -73,8 +72,6 @@ mod tests { use crate::ext::ext_repository::tests::{mocked_record, MockRecord}; use crate::storage::query::condition::Parser; - use reduct_base::io::{ReadRecord, RecordMeta}; - use reduct_base::Labels; use rstest::rstest; use serde_json::json; @@ -138,31 +135,4 @@ mod tests { "ignore bad condition" ) } - - #[rstest] - fn conflict(mut mocked_record: Box) { - let mut filter = ExtWhenFilter::new( - Some( - Parser::new() - .parse(&json!({"$and": [true, "@key1"]})) - .unwrap(), - ), - true, - ); - - let meta = RecordMeta::builder() - .labels(Labels::from_iter(vec![( - "key1".to_string(), - "value1".to_string(), - )])) - .computed_labels(mocked_record.meta().computed_labels().clone()) - .build(); - - *mocked_record.meta_mut() = meta; - - assert_eq!( - filter.filter_record(mocked_record).unwrap().err().unwrap(), - conflict!("Computed label '@key1' already exists") - ) - } } diff --git a/reductstore/src/storage/query/condition.rs b/reductstore/src/storage/query/condition.rs index d130d2d84..4ce180267 100644 --- a/reductstore/src/storage/query/condition.rs +++ b/reductstore/src/storage/query/condition.rs @@ -6,6 +6,7 @@ use reduct_base::error::ReductError; use std::collections::HashMap; use std::fmt::Debug; +mod computed_reference; mod constant; mod operators; mod parser; @@ -19,22 +20,18 @@ mod value; pub(crate) struct Context<'a> { timestamp: u64, labels: HashMap<&'a str, &'a str>, - stage: EvaluationStage, + computed_labels: HashMap<&'a str, &'a str>, } - -#[derive(PartialEq, Debug, Default, Clone)] -pub(crate) enum EvaluationStage { - #[default] - Retrieve, - Compute, -} - impl<'a> Context<'a> { - pub fn new(timestamp: u64, labels: HashMap<&'a str, &'a str>, stage: EvaluationStage) -> Self { + pub fn new( + timestamp: u64, + labels: HashMap<&'a str, &'a str>, + computed_labels: HashMap<&'a str, &'a str>, + ) -> Self { Context { timestamp, labels, - stage, + computed_labels, } } } @@ -46,22 +43,8 @@ pub(crate) trait Node { /// Evaluates the node in the given context. fn apply(&mut self, context: &Context) -> Result; - fn operands(&self) -> &Vec; - /// Returns a string representation of the node. fn print(&self) -> String; - - fn stage(&self) -> &EvaluationStage { - if self - .operands() - .iter() - .any(|o| o.stage() == &EvaluationStage::Compute) - { - &EvaluationStage::Compute - } else { - &EvaluationStage::Retrieve - } - } } pub(crate) trait Boxed: Node { diff --git a/reductstore/src/storage/query/condition/computed_reference.rs b/reductstore/src/storage/query/condition/computed_reference.rs new file mode 100644 index 000000000..1eef37de8 --- /dev/null +++ b/reductstore/src/storage/query/condition/computed_reference.rs @@ -0,0 +1,72 @@ +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 + +use crate::storage::query::condition::value::Value; +use crate::storage::query::condition::{BoxedNode, Context, Node}; +use reduct_base::error::ReductError; +use reduct_base::not_found; + +/// A node representing a reference to a label in the context. +pub(super) struct ComputedReference { + name: String, +} + +impl Node for ComputedReference { + fn apply(&mut self, context: &Context) -> Result { + let label_value = context + .computed_labels + .get(self.name.as_str()) + .ok_or(not_found!("Reference '@{}' not found", self.name))?; + + let value = Value::parse(label_value); + Ok(value) + } + + fn print(&self) -> String { + format!("CompRef({})", self.name) + } +} + +impl ComputedReference { + pub fn new(name: String) -> Self { + ComputedReference { name } + } + + pub fn boxed(name: String) -> BoxedNode { + Box::new(ComputedReference::new(name)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::query::condition::value::Value; + + #[test] + fn apply() { + let mut reference = ComputedReference::new("label".to_string()); + let mut context = Context::default(); + context.computed_labels.insert("label", "true"); + let result = reference.apply(&context).unwrap(); + assert_eq!(result, Value::Bool(true)); + } + + #[test] + fn apply_not_found() { + let mut reference = ComputedReference::new("label".to_string()); + let context = Context::default(); + let result = reference.apply(&context); + assert!(result + .err() + .unwrap() + .to_string() + .contains("Reference '@label' not found")); + } + + #[test] + fn print() { + let reference = ComputedReference::new("label".to_string()); + let result = reference.print(); + assert_eq!(result, "CompRef(label)"); + } +} diff --git a/reductstore/src/storage/query/condition/constant.rs b/reductstore/src/storage/query/condition/constant.rs index 9ed6ee59b..3bde42549 100644 --- a/reductstore/src/storage/query/condition/constant.rs +++ b/reductstore/src/storage/query/condition/constant.rs @@ -2,13 +2,12 @@ // Licensed under the Business Source License 1.1 use crate::storage::query::condition::value::Value; -use crate::storage::query::condition::{BoxedNode, Context, EvaluationStage, Node}; +use crate::storage::query::condition::{BoxedNode, Context, Node}; use reduct_base::error::ReductError; /// A node representing a constant value. pub(super) struct Constant { value: Value, - operands: Vec, } impl Node for Constant { @@ -16,25 +15,14 @@ impl Node for Constant { Ok(self.value.clone()) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("{:?}", self.value) } - - fn stage(&self) -> &EvaluationStage { - &EvaluationStage::Retrieve - } } impl Constant { pub fn new(value: Value) -> Self { - Constant { - value, - operands: vec![], - } + Constant { value } } pub fn boxed(value: Value) -> BoxedNode { @@ -46,7 +34,6 @@ impl From for Constant { fn from(value: bool) -> Self { Constant { value: Value::Bool(value), - operands: vec![], } } } @@ -74,13 +61,6 @@ mod tests { assert_eq!(result, "Bool(true)"); } - #[test] - fn operands() { - let constant = Constant::new(Value::Bool(true)); - let result = constant.operands(); - assert_eq!(result.len(), 0); - } - #[test] fn from_bool() { let mut constant = Constant::from(true); diff --git a/reductstore/src/storage/query/condition/operators/aggregation/each_n.rs b/reductstore/src/storage/query/condition/operators/aggregation/each_n.rs index c9ea44ef0..0b9e35cc4 100644 --- a/reductstore/src/storage/query/condition/operators/aggregation/each_n.rs +++ b/reductstore/src/storage/query/condition/operators/aggregation/each_n.rs @@ -48,10 +48,6 @@ impl Node for EachN { } } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("EachN({:?})", self.operands[0]) } diff --git a/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs b/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs index d5b9c2dae..9a89af79a 100644 --- a/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs +++ b/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs @@ -49,10 +49,6 @@ impl Node for EachT { Ok(Value::Bool(ret)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("EachT({:?})", self.operands[0]) } diff --git a/reductstore/src/storage/query/condition/operators/aggregation/limit.rs b/reductstore/src/storage/query/condition/operators/aggregation/limit.rs index fe81110e4..ccf39aba9 100644 --- a/reductstore/src/storage/query/condition/operators/aggregation/limit.rs +++ b/reductstore/src/storage/query/condition/operators/aggregation/limit.rs @@ -38,10 +38,6 @@ impl Node for Limit { Ok(Value::Bool(true)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Limit({:?})", self.operands[0]) } diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/abs.rs b/reductstore/src/storage/query/condition/operators/arithmetic/abs.rs index 8b59f0790..185aabb87 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/abs.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/abs.rs @@ -21,10 +21,6 @@ impl Node for Abs { fn print(&self) -> String { format!("Abs({:?})", self.operands[0]) } - - fn operands(&self) -> &Vec { - &self.operands - } } impl Boxed for Abs { diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/add.rs b/reductstore/src/storage/query/condition/operators/arithmetic/add.rs index cba41dbb3..967abd188 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/add.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/add.rs @@ -26,10 +26,6 @@ impl Node for Add { Ok(sum.unwrap_or(Value::Int(0))) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Add({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/div.rs b/reductstore/src/storage/query/condition/operators/arithmetic/div.rs index 8984ff704..34bc3c27f 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/div.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/div.rs @@ -19,10 +19,6 @@ impl Node for Div { value_1.divide(value_2) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Div({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/div_num.rs b/reductstore/src/storage/query/condition/operators/arithmetic/div_num.rs index b8a644515..a18a8d5d8 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/div_num.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/div_num.rs @@ -19,10 +19,6 @@ impl Node for DivNum { value_1.divide_num(value_2) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("DivNum({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/mult.rs b/reductstore/src/storage/query/condition/operators/arithmetic/mult.rs index 741ed365c..5718b30c5 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/mult.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/mult.rs @@ -26,10 +26,6 @@ impl Node for Mult { Ok(prod.unwrap_or(Value::Int(0))) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Mult({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/rem.rs b/reductstore/src/storage/query/condition/operators/arithmetic/rem.rs index c13bc31da..4ec7e2bd5 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/rem.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/rem.rs @@ -19,10 +19,6 @@ impl Node for Rem { value_1.remainder(value_2) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Rem({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/arithmetic/sub.rs b/reductstore/src/storage/query/condition/operators/arithmetic/sub.rs index b436420d2..906551f51 100644 --- a/reductstore/src/storage/query/condition/operators/arithmetic/sub.rs +++ b/reductstore/src/storage/query/condition/operators/arithmetic/sub.rs @@ -19,10 +19,6 @@ impl Node for Sub { value_1.subtract(value_2) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Sub({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/comparison/eq.rs b/reductstore/src/storage/query/condition/operators/comparison/eq.rs index 8732c945f..059e48412 100644 --- a/reductstore/src/storage/query/condition/operators/comparison/eq.rs +++ b/reductstore/src/storage/query/condition/operators/comparison/eq.rs @@ -18,10 +18,6 @@ impl Node for Eq { Ok(Value::Bool(value_1 == value_2)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Eq({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/comparison/gt.rs b/reductstore/src/storage/query/condition/operators/comparison/gt.rs index eea0a3b61..3f00b8c1d 100644 --- a/reductstore/src/storage/query/condition/operators/comparison/gt.rs +++ b/reductstore/src/storage/query/condition/operators/comparison/gt.rs @@ -18,10 +18,6 @@ impl Node for Gt { Ok(Value::Bool(value_1 > value_2)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Gt({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/comparison/gte.rs b/reductstore/src/storage/query/condition/operators/comparison/gte.rs index 8fc154302..597ff7d84 100644 --- a/reductstore/src/storage/query/condition/operators/comparison/gte.rs +++ b/reductstore/src/storage/query/condition/operators/comparison/gte.rs @@ -18,10 +18,6 @@ impl Node for Gte { Ok(Value::Bool(value_1 >= value_2)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Gte({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/comparison/lt.rs b/reductstore/src/storage/query/condition/operators/comparison/lt.rs index 1e841f612..74ab51497 100644 --- a/reductstore/src/storage/query/condition/operators/comparison/lt.rs +++ b/reductstore/src/storage/query/condition/operators/comparison/lt.rs @@ -18,10 +18,6 @@ impl Node for Lt { Ok(Value::Bool(value_1 < value_2)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Lt({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/comparison/lte.rs b/reductstore/src/storage/query/condition/operators/comparison/lte.rs index b83e6cc47..44cc628c7 100644 --- a/reductstore/src/storage/query/condition/operators/comparison/lte.rs +++ b/reductstore/src/storage/query/condition/operators/comparison/lte.rs @@ -18,10 +18,6 @@ impl Node for Lte { Ok(Value::Bool(value_1 <= value_2)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Lte({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/comparison/ne.rs b/reductstore/src/storage/query/condition/operators/comparison/ne.rs index d847d43af..94e7363b9 100644 --- a/reductstore/src/storage/query/condition/operators/comparison/ne.rs +++ b/reductstore/src/storage/query/condition/operators/comparison/ne.rs @@ -18,10 +18,6 @@ impl Node for Ne { Ok(Value::Bool(value_1 != value_2)) } - fn operands(&self) -> &Vec { - self.operands.as_ref() - } - fn print(&self) -> String { format!("Ne({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/logical/all_of.rs b/reductstore/src/storage/query/condition/operators/logical/all_of.rs index 34d104b0e..53bccea43 100644 --- a/reductstore/src/storage/query/condition/operators/logical/all_of.rs +++ b/reductstore/src/storage/query/condition/operators/logical/all_of.rs @@ -22,10 +22,6 @@ impl Node for AllOf { Ok(Value::Bool(true)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("AllOf({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/logical/any_of.rs b/reductstore/src/storage/query/condition/operators/logical/any_of.rs index 93b15dd05..90cb4eb26 100644 --- a/reductstore/src/storage/query/condition/operators/logical/any_of.rs +++ b/reductstore/src/storage/query/condition/operators/logical/any_of.rs @@ -22,10 +22,6 @@ impl Node for AnyOf { Ok(Value::Bool(false)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("AnyOf({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/logical/in.rs b/reductstore/src/storage/query/condition/operators/logical/in.rs index f62ea76fc..5b54afa2b 100644 --- a/reductstore/src/storage/query/condition/operators/logical/in.rs +++ b/reductstore/src/storage/query/condition/operators/logical/in.rs @@ -23,10 +23,6 @@ impl Node for In { Ok(Value::Bool(false)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!( "In({:?}, [{:}])", diff --git a/reductstore/src/storage/query/condition/operators/logical/nin.rs b/reductstore/src/storage/query/condition/operators/logical/nin.rs index d8c15cfa6..1944c5339 100644 --- a/reductstore/src/storage/query/condition/operators/logical/nin.rs +++ b/reductstore/src/storage/query/condition/operators/logical/nin.rs @@ -22,9 +22,6 @@ impl Node for Nin { Ok(Value::Bool(true)) } - fn operands(&self) -> &Vec { - &self.operands - } fn print(&self) -> String { format!( diff --git a/reductstore/src/storage/query/condition/operators/logical/none_of.rs b/reductstore/src/storage/query/condition/operators/logical/none_of.rs index 45b4b1af1..76ea9c61e 100644 --- a/reductstore/src/storage/query/condition/operators/logical/none_of.rs +++ b/reductstore/src/storage/query/condition/operators/logical/none_of.rs @@ -22,10 +22,6 @@ impl Node for NoneOf { Ok(Value::Bool(true)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("NoneOf({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/logical/one_of.rs b/reductstore/src/storage/query/condition/operators/logical/one_of.rs index 92f57e933..1029bc895 100644 --- a/reductstore/src/storage/query/condition/operators/logical/one_of.rs +++ b/reductstore/src/storage/query/condition/operators/logical/one_of.rs @@ -23,10 +23,6 @@ impl Node for OneOf { Ok(Value::Bool(count == 1)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("OneOf({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/misc/cast.rs b/reductstore/src/storage/query/condition/operators/misc/cast.rs index b680126e0..8f74ea2b7 100644 --- a/reductstore/src/storage/query/condition/operators/misc/cast.rs +++ b/reductstore/src/storage/query/condition/operators/misc/cast.rs @@ -18,10 +18,6 @@ impl Node for Cast { op.cast(type_name.as_str()) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Cast({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/misc/exists.rs b/reductstore/src/storage/query/condition/operators/misc/exists.rs index 8dd48e08a..0eee8639a 100644 --- a/reductstore/src/storage/query/condition/operators/misc/exists.rs +++ b/reductstore/src/storage/query/condition/operators/misc/exists.rs @@ -22,10 +22,6 @@ impl Node for Exists { Ok(Value::Bool(true)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Exists({:?})", self.operands) } diff --git a/reductstore/src/storage/query/condition/operators/misc/ref.rs b/reductstore/src/storage/query/condition/operators/misc/ref.rs index d6eff7464..1ee710bbf 100644 --- a/reductstore/src/storage/query/condition/operators/misc/ref.rs +++ b/reductstore/src/storage/query/condition/operators/misc/ref.rs @@ -20,10 +20,6 @@ impl Node for Ref { ) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Ref({:?})", self.operands[0]) } diff --git a/reductstore/src/storage/query/condition/operators/misc/timestamp.rs b/reductstore/src/storage/query/condition/operators/misc/timestamp.rs index 7fd0d9ff7..b51e3c72d 100644 --- a/reductstore/src/storage/query/condition/operators/misc/timestamp.rs +++ b/reductstore/src/storage/query/condition/operators/misc/timestamp.rs @@ -7,19 +7,13 @@ use reduct_base::error::ReductError; use reduct_base::unprocessable_entity; /// A node representing the timestamp of a current record in a query. -pub(crate) struct Timestamp { - operands: Vec, -} +pub(crate) struct Timestamp {} impl Node for Timestamp { fn apply(&mut self, context: &Context) -> Result { Ok(Value::Int(context.timestamp as i64)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { "Timestamp()".to_string() } @@ -31,13 +25,13 @@ impl Boxed for Timestamp { return Err(unprocessable_entity!("$timestamp requires no operands")); } - Ok(Box::new(Timestamp::new(operands))) + Ok(Box::new(Timestamp::new())) } } impl Timestamp { - pub fn new(operands: Vec) -> Self { - Self { operands } + pub fn new() -> Self { + Self {} } } @@ -51,7 +45,7 @@ mod tests { #[rstest] fn apply_ok() { - let mut op = Timestamp::new(vec![]); + let mut op = Timestamp::new(); let mut context = Context::default(); context.timestamp = 1234567890; @@ -69,7 +63,7 @@ mod tests { #[rstest] fn print() { - let and = Timestamp::new(vec![]); + let and = Timestamp::new(); assert_eq!(and.print(), "Timestamp()".to_string()); } } diff --git a/reductstore/src/storage/query/condition/operators/string/contains.rs b/reductstore/src/storage/query/condition/operators/string/contains.rs index 7335b6012..e73d8fa6c 100644 --- a/reductstore/src/storage/query/condition/operators/string/contains.rs +++ b/reductstore/src/storage/query/condition/operators/string/contains.rs @@ -18,10 +18,6 @@ impl Node for Contains { Ok(Value::Bool(value_1.contains(value_2)?)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("Contains({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/string/ends_with.rs b/reductstore/src/storage/query/condition/operators/string/ends_with.rs index f7f3aace7..4a60a4a71 100644 --- a/reductstore/src/storage/query/condition/operators/string/ends_with.rs +++ b/reductstore/src/storage/query/condition/operators/string/ends_with.rs @@ -18,10 +18,6 @@ impl Node for EndsWith { Ok(Value::Bool(value_1.ends_with(value_2)?)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("EndsWith({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/operators/string/starts_with.rs b/reductstore/src/storage/query/condition/operators/string/starts_with.rs index e5e809614..e53a16190 100644 --- a/reductstore/src/storage/query/condition/operators/string/starts_with.rs +++ b/reductstore/src/storage/query/condition/operators/string/starts_with.rs @@ -18,10 +18,6 @@ impl Node for StartsWith { Ok(Value::Bool(value_1.starts_with(value_2)?)) } - fn operands(&self) -> &Vec { - &self.operands - } - fn print(&self) -> String { format!("StartsWith({:?}, {:?})", self.operands[0], self.operands[1]) } diff --git a/reductstore/src/storage/query/condition/parser.rs b/reductstore/src/storage/query/condition/parser.rs index baa90b683..d1c9326d3 100644 --- a/reductstore/src/storage/query/condition/parser.rs +++ b/reductstore/src/storage/query/condition/parser.rs @@ -1,6 +1,7 @@ // Copyright 2024 ReductSoftware UG // Licensed under the Business Source License 1.1 +use crate::storage::query::condition::computed_reference::ComputedReference; use crate::storage::query::condition::constant::Constant; use crate::storage::query::condition::operators::aggregation::{EachN, EachT, Limit}; use crate::storage::query::condition::operators::arithmetic::{ @@ -12,7 +13,7 @@ use crate::storage::query::condition::operators::misc::{Cast, Exists, Ref, Times use crate::storage::query::condition::operators::string::{Contains, EndsWith, StartsWith}; use crate::storage::query::condition::reference::Reference; use crate::storage::query::condition::value::Value; -use crate::storage::query::condition::{Boxed, BoxedNode, Context, EvaluationStage, Node}; +use crate::storage::query::condition::{Boxed, BoxedNode}; use reduct_base::error::ReductError; use reduct_base::unprocessable_entity; use serde_json::{Map, Number, Value as JsonValue}; @@ -20,71 +21,6 @@ use serde_json::{Map, Number, Value as JsonValue}; /// Parses a JSON object into a condition tree. pub(crate) struct Parser {} -/// A node in a condition tree. -/// -/// It evaluates the node in the given context on different stages. -struct StagedAllOf { - operands: Vec, -} - -impl Node for StagedAllOf { - fn apply(&mut self, context: &Context) -> Result { - match context.stage { - EvaluationStage::Retrieve => self.apply_retrieve_stage(context), - EvaluationStage::Compute => self.apply_compute_stage(context), - } - } - - fn operands(&self) -> &Vec { - &self.operands - } - - fn print(&self) -> String { - format!("AllOf({:?})", self.operands) - } -} - -impl Boxed for StagedAllOf { - fn boxed(operands: Vec) -> Result { - Ok(Box::new(Self { operands })) - } -} - -impl StagedAllOf { - fn apply_retrieve_stage(&mut self, context: &Context) -> Result { - // In the retrieve stage, we evaluate all operands before the first compute-staged one - for operand in &mut self.operands { - if operand.stage() == &EvaluationStage::Compute { - break; - } - - let value = operand.apply(context)?; - if !value.as_bool()? { - return Ok(Value::Bool(false)); - } - } - - Ok(Value::Bool(true)) - } - - fn apply_compute_stage(&mut self, context: &Context) -> Result { - // In the compute stage, we evaluate all operands after the first compute-staged one - let mut compute_operand_detected = false; - for operand in &mut self.operands { - if operand.stage() == &EvaluationStage::Compute || compute_operand_detected { - compute_operand_detected = true; - - let value = operand.apply(context)?; - if !value.as_bool()? { - return Ok(Value::Bool(false)); - } - } - } - - Ok(Value::Bool(true)) - } -} - impl Parser { /// Parses a JSON object into a condition tree. /// @@ -98,7 +34,7 @@ impl Parser { /// The root node is a `StagedAllOf` that aggregates all expressions pub fn parse(&self, json: &JsonValue) -> Result { let expressions = Self::parse_recursively(json)?; - Ok(StagedAllOf::boxed(expressions)?) + Ok(AllOf::boxed(expressions)?) } fn parse_recursively(json: &JsonValue) -> Result, ReductError> { @@ -153,15 +89,9 @@ impl Parser { fn parse_string(value: &str) -> Result, ReductError> { if value.starts_with("&") { - Ok(vec![Reference::boxed( - value[1..].to_string(), - EvaluationStage::Retrieve, - )]) + Ok(vec![Reference::boxed(value[1..].to_string())]) } else if value.starts_with("@") { - Ok(vec![Reference::boxed( - value[1..].to_string(), - EvaluationStage::Compute, - )]) + Ok(vec![ComputedReference::boxed(value[1..].to_string())]) } else if value.starts_with("$") { // operator without operands (nullary) Ok(vec![Self::parse_operator(value, vec![])?]) @@ -285,11 +215,7 @@ mod tests { "&label": {"$gt": 10} }); let mut node = parser.parse(&json).unwrap(); - let context = Context::new( - 0, - HashMap::from_iter(vec![("label", "20")]), - EvaluationStage::Retrieve, - ); + let context = Context::new(0, HashMap::from_iter(vec![("label", "20")]), HashMap::new()); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -333,7 +259,7 @@ mod tests { let context = Context::new( 0, HashMap::from_iter(vec![("label", "true")]), - EvaluationStage::Retrieve, + HashMap::new(), ); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -493,69 +419,6 @@ mod tests { } } - mod staged_all_of { - use super::*; - use rstest::rstest; - - #[rstest] - fn test_staged_all_of() { - let operands: Vec = vec![ - Constant::boxed(Value::Bool(true)), - Constant::boxed(Value::Bool(false)), - ]; - let staged_all_of = StagedAllOf::boxed(operands); - assert_eq!( - staged_all_of.unwrap().print(), - "AllOf([Bool(true), Bool(false)])" - ); - } - - #[rstest] - fn test_run_only_staged_all_of() { - let operands: Vec = vec![ - Constant::boxed(Value::Bool(false)), // ignored because not in stage - ]; - - let mut staged_all_of = StagedAllOf::boxed(operands).unwrap(); - let context = Context::new(0, HashMap::new(), EvaluationStage::Compute); - assert_eq!(staged_all_of.apply(&context).unwrap(), Value::Bool(true)); - } - - #[rstest] - fn test_all_condition_after_compute_operand() { - let operands: Vec = vec![ - Reference::boxed("label".to_string(), EvaluationStage::Compute), - Constant::boxed(Value::Bool(false)), - ]; - - let mut staged_all_of = StagedAllOf::boxed(operands).unwrap(); - let context = Context::new( - 0, - HashMap::from_iter(vec![("label", "true")]), - EvaluationStage::Compute, - ); - assert_eq!(staged_all_of.apply(&context).unwrap(), Value::Bool(false), - "Must be false because the last retrieved value is false and it is used on the Compute stage"); - } - - #[rstest] - fn test_all_condition_before_compute_operand() { - let operands: Vec = vec![ - Constant::boxed(Value::Bool(true)), - Reference::boxed("label".to_string(), EvaluationStage::Compute), - ]; - - let mut staged_all_of = StagedAllOf::boxed(operands).unwrap(); - let context = Context::new( - 0, - HashMap::from_iter(vec![("label", "false")]), - EvaluationStage::Retrieve, - ); - assert_eq!(staged_all_of.apply(&context).unwrap(), Value::Bool(true), - "Must be true because the last retrieved value, and compute-staged label is ignored"); - } - } - #[fixture] fn parser() -> Parser { Parser::new() diff --git a/reductstore/src/storage/query/condition/reference.rs b/reductstore/src/storage/query/condition/reference.rs index e00d50388..64858aa80 100644 --- a/reductstore/src/storage/query/condition/reference.rs +++ b/reductstore/src/storage/query/condition/reference.rs @@ -2,15 +2,13 @@ // Licensed under the Business Source License 1.1 use crate::storage::query::condition::value::Value; -use crate::storage::query::condition::{BoxedNode, Context, EvaluationStage, Node}; +use crate::storage::query::condition::{BoxedNode, Context, Node}; use reduct_base::error::ReductError; use reduct_base::not_found; /// A node representing a reference to a label in the context. pub(super) struct Reference { name: String, - stage: EvaluationStage, - operands: Vec, } impl Node for Reference { @@ -24,30 +22,18 @@ impl Node for Reference { Ok(value) } - fn operands(&self) -> &Vec { - &self.operands - } - - fn stage(&self) -> &EvaluationStage { - &self.stage - } - fn print(&self) -> String { format!("Ref({})", self.name) } } impl Reference { - pub fn new(name: String, stage: EvaluationStage) -> Self { - Reference { - name, - stage, - operands: vec![], - } + pub fn new(name: String) -> Self { + Reference { name } } - pub fn boxed(name: String, stage: EvaluationStage) -> BoxedNode { - Box::new(Reference::new(name, stage)) + pub fn boxed(name: String) -> BoxedNode { + Box::new(Reference::new(name)) } } @@ -55,11 +41,10 @@ impl Reference { mod tests { use super::*; use crate::storage::query::condition::value::Value; - use rstest::rstest; #[test] fn apply() { - let mut reference = Reference::new("label".to_string(), EvaluationStage::Retrieve); + let mut reference = Reference::new("label".to_string()); let mut context = Context::default(); context.labels.insert("label", "true"); let result = reference.apply(&context).unwrap(); @@ -68,7 +53,7 @@ mod tests { #[test] fn apply_not_found() { - let mut reference = Reference::new("label".to_string(), EvaluationStage::Retrieve); + let mut reference = Reference::new("label".to_string()); let context = Context::default(); let result = reference.apply(&context); assert!(result @@ -80,24 +65,8 @@ mod tests { #[test] fn print() { - let reference = Reference::new("label".to_string(), EvaluationStage::Retrieve); + let reference = Reference::new("label".to_string()); let result = reference.print(); assert_eq!(result, "Ref(label)"); } - - #[test] - fn operands() { - let reference = Reference::new("label".to_string(), EvaluationStage::Retrieve); - let result = reference.operands(); - assert_eq!(result.len(), 0); - } - - #[rstest] - #[case(EvaluationStage::Retrieve)] - #[case(EvaluationStage::Compute)] - fn test_stage(#[case] stage: EvaluationStage) { - let reference = Reference::new("label".to_string(), stage.clone()); - let result = reference.stage(); - assert_eq!(result, &stage); - } } diff --git a/reductstore/src/storage/query/filters/when.rs b/reductstore/src/storage/query/filters/when.rs index d04540620..d885ddb44 100644 --- a/reductstore/src/storage/query/filters/when.rs +++ b/reductstore/src/storage/query/filters/when.rs @@ -1,7 +1,7 @@ // Copyright 2023-2024 ReductSoftware UG // Licensed under the Business Source License 1.1 -use crate::storage::query::condition::{BoxedNode, Context, EvaluationStage}; +use crate::storage::query::condition::{BoxedNode, Context}; use crate::storage::query::filters::{RecordFilter, RecordMeta}; use reduct_base::error::ReductError; @@ -25,7 +25,11 @@ impl RecordFilter for WhenFilter { .iter() .map(|(k, v)| (k.as_str(), v.as_str())) .collect(), - EvaluationStage::Retrieve, + record + .computed_labels() + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(), ); Ok(self.condition.apply(&context)?.as_bool()?) } From 89cea0bbe71386a978d2fa41c71a283e25158200 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jun 2025 09:42:00 +0200 Subject: [PATCH 27/93] Bump tower-http from 0.6.4 to 0.6.5 (#842) Bumps [tower-http](https://github.com/tower-rs/tower-http) from 0.6.4 to 0.6.5. - [Release notes](https://github.com/tower-rs/tower-http/releases) - [Commits](https://github.com/tower-rs/tower-http/compare/tower-http-0.6.4...tower-http-0.6.5) --- updated-dependencies: - dependency-name: tower-http dependency-version: 0.6.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29f784d36..7b54d78d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2792,9 +2792,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "5cc2d9e086a412a451384326f521c8123a99a466b329941a9403696bff9b0da2" dependencies = [ "bitflags", "bytes", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index ddf52e9c8..6454895f3 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -55,7 +55,7 @@ base64 = "0.22.1" ring = "0.17.12" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } async-stream = "0.3.6" -tower-http = { version = "0.6.4", features = ["cors"] } +tower-http = { version = "0.6.5", features = ["cors"] } crc64fast = "1.1.0" rustls = "0.23.27" byteorder = "1.5.0" From f190a1bf74d66630ea2def7de630ccf1a778a21b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jun 2025 09:42:26 +0200 Subject: [PATCH 28/93] Bump dlopen2 from 0.7.0 to 0.8.0 (#840) Bumps [dlopen2](https://github.com/OpenByteDev/dlopen2) from 0.7.0 to 0.8.0. - [Commits](https://github.com/OpenByteDev/dlopen2/commits) --- updated-dependencies: - dependency-name: dlopen2 dependency-version: 0.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b54d78d2..c64c5da85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "dlopen2" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1297103d2bbaea85724fcee6294c2d50b1081f9ad47d0f6f6f61eda65315a6" +checksum = "b54f373ccf864bf587a89e880fb7610f8d73f3045f13580948ccbcaff26febff" dependencies = [ "dlopen2_derive", "libc", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 6454895f3..059f6f75c 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -60,7 +60,7 @@ crc64fast = "1.1.0" rustls = "0.23.27" byteorder = "1.5.0" crossbeam-channel = "0.5.15" -dlopen2 = "0.7.0" +dlopen2 = "0.8.0" log = "0.4" prost = "0.13.1" From e45283e8e1c40c8be7faf79efdd06def740a4c40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Jun 2025 09:43:09 +0200 Subject: [PATCH 29/93] Bump reqwest from 0.12.15 to 0.12.18 (#841) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.15 to 0.12.18. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.15...v0.12.18) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.12.18 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 168 +++++++++++++---------------------------- reductstore/Cargo.toml | 4 +- 2 files changed, 53 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c64c5da85..40f105891 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,7 +320,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1061,21 +1061,26 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 0.26.8", ] [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -1301,6 +1306,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -1390,7 +1405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1652,7 +1667,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -2119,9 +2134,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" dependencies = [ "base64 0.22.1", "bytes", @@ -2143,7 +2158,6 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", @@ -2153,14 +2167,14 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", - "windows-registry", + "webpki-roots 1.0.0", ] [[package]] @@ -2495,9 +2509,9 @@ checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2798,8 +2812,12 @@ checksum = "5cc2d9e086a412a451384326f521c8123a99a466b329941a9403696bff9b0da2" dependencies = [ "bitflags", "bytes", + "futures-util", "http", + "http-body", + "iri-string", "pin-project-lite", + "tower", "tower-layer", "tower-service", ] @@ -3104,6 +3122,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.4.2" @@ -3144,7 +3171,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3153,42 +3180,13 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" -[[package]] -name = "windows-registry" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" -dependencies = [ - "windows-result", - "windows-strings", - "windows-targets 0.53.0", -] - -[[package]] -name = "windows-result" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-link", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3197,7 +3195,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -3206,30 +3204,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -3238,96 +3220,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - [[package]] name = "winnow" version = "0.5.40" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 059f6f75c..b24622a8e 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -66,7 +66,7 @@ prost = "0.13.1" [build-dependencies] prost-build = "0.13.1" -reqwest = { version = "0.12.15", default-features = false, features = ["rustls-tls", "blocking", "json"] } +reqwest = { version = "0.12.18", default-features = false, features = ["rustls-tls", "blocking", "json"] } chrono = "0.4.41" serde_json = "1.0.140" @@ -75,7 +75,7 @@ mockall = "0.13.1" rstest = "0.25.0" serial_test = "3.2.0" test-log = "0.2.17" -reqwest = { version = "0.12.15", default-features = false, features = ["rustls-tls", "blocking"] } +reqwest = { version = "0.12.18", default-features = false, features = ["rustls-tls", "blocking"] } assert_matches = "1.5" [package.metadata.docs.rs] From dbb8c1a5306b5d75ae075515fd804895e21911ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 08:51:23 +0200 Subject: [PATCH 30/93] Bump tower-http from 0.6.5 to 0.6.6 (#844) Bumps [tower-http](https://github.com/tower-rs/tower-http) from 0.6.5 to 0.6.6. - [Release notes](https://github.com/tower-rs/tower-http/releases) - [Commits](https://github.com/tower-rs/tower-http/compare/tower-http-0.6.5...tower-http-0.6.6) --- updated-dependencies: - dependency-name: tower-http dependency-version: 0.6.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40f105891..80df88198 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2806,9 +2806,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc2d9e086a412a451384326f521c8123a99a466b329941a9403696bff9b0da2" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags", "bytes", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index b24622a8e..78d2aaa8e 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -55,7 +55,7 @@ base64 = "0.22.1" ring = "0.17.12" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } async-stream = "0.3.6" -tower-http = { version = "0.6.5", features = ["cors"] } +tower-http = { version = "0.6.6", features = ["cors"] } crc64fast = "1.1.0" rustls = "0.23.27" byteorder = "1.5.0" From b6daf8a80a7c58b77fd05af3ceda6eacb2f113a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 08:51:36 +0200 Subject: [PATCH 31/93] Bump reqwest from 0.12.18 to 0.12.19 (#845) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.18 to 0.12.19. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.18...v0.12.19) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.12.19 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80df88198..0fedf8bc1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2134,9 +2134,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.18" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" dependencies = [ "base64 0.22.1", "bytes", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 78d2aaa8e..fe47a2ada 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -66,7 +66,7 @@ prost = "0.13.1" [build-dependencies] prost-build = "0.13.1" -reqwest = { version = "0.12.18", default-features = false, features = ["rustls-tls", "blocking", "json"] } +reqwest = { version = "0.12.19", default-features = false, features = ["rustls-tls", "blocking", "json"] } chrono = "0.4.41" serde_json = "1.0.140" @@ -75,7 +75,7 @@ mockall = "0.13.1" rstest = "0.25.0" serial_test = "3.2.0" test-log = "0.2.17" -reqwest = { version = "0.12.18", default-features = false, features = ["rustls-tls", "blocking"] } +reqwest = { version = "0.12.19", default-features = false, features = ["rustls-tls", "blocking"] } assert_matches = "1.5" [package.metadata.docs.rs] From 68b0ee363fd4240fea3acbf2c95ffcff93c96e64 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Mon, 9 Jun 2025 11:39:29 +0200 Subject: [PATCH 32/93] Fix crash if RS_API_PATH has wrong format (#846) * normalize api path * update CHANGELOG --- CHANGELOG.md | 4 ++++ reductstore/src/cfg.rs | 26 ++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b86cba9..7891013ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## Fixed + +- Fix crash if RS_API_PATH has wrong format, [PR-846](https://github.com/reductstore/reductstore/pull/846) + ## [1.15.4] - 2025-05-28 ### Fixed diff --git a/reductstore/src/cfg.rs b/reductstore/src/cfg.rs index 8befea23c..5de28d635 100644 --- a/reductstore/src/cfg.rs +++ b/reductstore/src/cfg.rs @@ -53,11 +53,15 @@ impl Cfg { pub fn from_env(env_getter: EnvGetter) -> Self { let mut env = Env::new(env_getter); let port: u16 = env.get("RS_PORT", DEFAULT_PORT); + + let mut api_base_path = env.get("RS_API_BASE_PATH", "/".to_string()); + Self::normalize_url_path(&mut api_base_path); + let cfg = Cfg { log_level: env.get("RS_LOG_LEVEL", DEFAULT_LOG_LEVEL.to_string()), host: env.get("RS_HOST", DEFAULT_HOST.to_string()), port: port.clone(), - api_base_path: env.get("RS_API_BASE_PATH", "/".to_string()), + api_base_path, data_path: env.get("RS_DATA_PATH", "/data".to_string()), api_token: env.get_masked("RS_API_TOKEN", "".to_string()), cert_path: env.get_masked("RS_CERT_PATH", "".to_string()), @@ -76,6 +80,16 @@ impl Cfg { cfg } + fn normalize_url_path(api_base_path: &mut String) { + if !api_base_path.starts_with('/') { + api_base_path.insert(0, '/'); + } + + if !api_base_path.ends_with('/') { + api_base_path.push('/'); + } + } + pub fn build(&self) -> Result { let storage = Arc::new(self.provision_buckets()); let token_repo = self.provision_tokens(); @@ -217,17 +231,21 @@ mod tests { } #[rstest] - fn test_api_base_path(mut env_getter: MockEnvGetter) { + #[case("/api")] + #[case("/api/")] + #[case("api/")] + #[case("api")] + fn test_api_base_path(mut env_getter: MockEnvGetter, #[case] path: &str) { env_getter .expect_get() .with(eq("RS_API_BASE_PATH")) .times(1) - .return_const(Ok("/api".to_string())); + .return_const(Ok(path.to_string())); env_getter .expect_get() .return_const(Err(VarError::NotPresent)); let cfg = Cfg::from_env(env_getter); - assert_eq!(cfg.api_base_path, "/api"); + assert_eq!(cfg.api_base_path, "/api/"); } #[rstest] From b8716a1e1a37853f5a35ab6411fb876ae609aeec Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 10 Jun 2025 17:32:59 +0200 Subject: [PATCH 33/93] Update web console up to 1.10.2 (#847) * update web console up to 1.10.2 * update CHANGELOG --- CHANGELOG.md | 4 ++++ reductstore/build.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7891013ee..e06905983 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix crash if RS_API_PATH has wrong format, [PR-846](https://github.com/reductstore/reductstore/pull/846) +## Changed + +- Update Web Console up to 1.10.2, [PR-847](https://github.com/reductstore/reductstore/pull/847) + ## [1.15.4] - 2025-05-28 ### Fixed diff --git a/reductstore/build.rs b/reductstore/build.rs index 9239dfda3..cc981402f 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -45,7 +45,7 @@ fn main() -> Result<(), Box> { #[cfg(feature = "web-console")] fn download_web_console() { - const WEB_CONSOLE_VERSION: &str = "v1.10.1"; + const WEB_CONSOLE_VERSION: &str = "v1.10.2"; let out_dir = env::var("OUT_DIR").unwrap(); let console_path = &format!("{}/console-{}.zip", out_dir, WEB_CONSOLE_VERSION); if Path::exists(Path::new(console_path)) { From 9aa9c5e1f9800a652463b67e0445550b013e4b55 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 10 Jun 2025 17:33:58 +0200 Subject: [PATCH 34/93] release v1.15.5 --- CHANGELOG.md | 6 ++++-- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e06905983..5b0d21941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## Fixed +## [1.15.5] - 2025-06-10 + +### Fixed - Fix crash if RS_API_PATH has wrong format, [PR-846](https://github.com/reductstore/reductstore/pull/846) -## Changed +### Changed - Update Web Console up to 1.10.2, [PR-847](https://github.com/reductstore/reductstore/pull/847) diff --git a/Cargo.lock b/Cargo.lock index bb8c7ab6a..659467341 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1995,7 +1995,7 @@ dependencies = [ [[package]] name = "reduct-base" -version = "1.15.4" +version = "1.15.5" dependencies = [ "async-trait", "bytes", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "reduct-macros" -version = "1.15.4" +version = "1.15.5" dependencies = [ "quote", "syn 2.0.101", @@ -2021,7 +2021,7 @@ dependencies = [ [[package]] name = "reductstore" -version = "1.15.4" +version = "1.15.5" dependencies = [ "assert_matches", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index a60ac5fb8..96220768c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.15.4" +version = "1.15.5" authors = ["Alexey Timin ", "ReductSoftware UG "] edition = "2021" rust-version = "1.85.0" From 4195b57fe308ebb2680aff8e06e6b66326825f08 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 12 Jun 2025 11:14:25 +0200 Subject: [PATCH 35/93] Fix $limit operator in extension context (#848) * fix handling of $limit operator * add tests * update CHANGELOG --- CHANGELOG.md | 1 + reductstore/src/api.rs | 29 ++++- reductstore/src/ext/ext_repository.rs | 103 ++++++++++++++++-- .../src/storage/entry/io/record_reader.rs | 7 +- 4 files changed, 126 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1927692d3..9e6f2ac0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fix hanging query request if no extension registered, [PR-830](https://github.com/reductstore/reductstore/pull/830) +- Fix `$limit` operator in extension context, [PR-848](https://github.com/reductstore/reductstore/pull/848) ## [1.15.5] - 2025-06-10 diff --git a/reductstore/src/api.rs b/reductstore/src/api.rs index 7c22bdb41..9fa1fa52b 100644 --- a/reductstore/src/api.rs +++ b/reductstore/src/api.rs @@ -26,6 +26,7 @@ use axum::{middleware::from_fn, Router}; use bucket::create_bucket_api_routes; use entry::create_entry_api_routes; use hyper::http::HeaderValue; +use log::warn; use middleware::{default_headers, print_statuses}; pub use reduct_base::error::ErrorCode; use reduct_base::error::ReductError as BaseHttpError; @@ -84,7 +85,14 @@ impl IntoResponse for HttpError { let body = format!("{{\"detail\": \"{}\"}}", converted_quotes); // its often easiest to implement `IntoResponse` by calling other implementations - let mut resp = (StatusCode::from_u16(err.status as u16).unwrap(), body).into_response(); + let http_code = if (err.status as i16) < 0 { + warn!("Invalid status code: {}", err.status); + StatusCode::INTERNAL_SERVER_ERROR + } else { + StatusCode::from_u16(err.status as u16).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) + }; + + let mut resp = (http_code, body).into_response(); resp.headers_mut() .insert("content-type", "application/json".parse().unwrap()); resp.headers_mut() @@ -203,6 +211,25 @@ mod tests { assert_eq!(body, Bytes::from(r#"{"detail": "Test error"}"#)) } + #[rstest] + #[tokio::test] + async fn test_no_http_error() { + let error = HttpError::new(ErrorCode::Interrupt, "Test error"); + let resp = error.into_response(); + assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + resp.headers().get("content-type").unwrap(), + HeaderValue::from_static("application/json") + ); + assert_eq!( + resp.headers().get("x-reduct-error").unwrap(), + HeaderValue::from_static("Test error") + ); + + let body: Bytes = to_bytes(resp.into_body(), 1000).await.unwrap(); + assert_eq!(body, Bytes::from(r#"{"detail": "Test error"}"#)) + } + #[rstest] #[tokio::test] async fn test_http_json_format() { diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index 14029c637..598b468b4 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -12,7 +12,7 @@ use crate::storage::query::QueryRx; use async_trait::async_trait; use dlopen2::wrapper::{Container, WrapperApi}; use futures_util::StreamExt; -use reduct_base::error::ErrorCode::NoContent; +use reduct_base::error::ErrorCode::{Interrupt, NoContent}; use reduct_base::error::ReductError; use reduct_base::ext::{ BoxedCommiter, BoxedProcessor, BoxedReadRecord, BoxedRecordStream, ExtSettings, IoExtension, @@ -223,14 +223,17 @@ impl ManageExtensions for ExtRepository { let record = result.unwrap(); return match query.condition_filter.filter_record(record) { - Some(result) => { - let record = match result { - Ok(record) => record, - Err(e) => return Some(Err(e)), - }; - - query.commiter.commit_record(record).await - } + Some(result) => match result { + Ok(record) => query.commiter.commit_record(record).await, + Err(e) => { + if e.status == Interrupt { + query.current_stream = None; + None + } else { + Some(Err(e)) + } + } + }, None => None, }; } else { @@ -281,8 +284,9 @@ pub(super) mod tests { use crate::storage::entry::RecordReader; use crate::storage::proto::Record; - use mockall::mock; + use async_stream::stream; use mockall::predicate::eq; + use mockall::{mock, predicate}; use prost_wkt_types::Timestamp; use reduct_base::io::{ReadChunk, ReadRecord, RecordMeta}; use reduct_base::msg::server_api::ServerInfo; @@ -743,6 +747,85 @@ pub(super) mod tests { } } + #[rstest] + #[tokio::test(flavor = "current_thread")] + async fn test_process_a_record_limit( + record_reader: RecordReader, + mut mock_ext: MockIoExtension, + mut processor: Box, + mut commiter: Box, + ) { + processor.expect_process_record().return_once(|_| { + let stream = stream! { + yield Ok(MockRecord::boxed("key", "val")); + yield Ok(MockRecord::boxed("key", "val")); + }; + Ok(Box::new(stream) as BoxedRecordStream) + }); + + commiter.expect_commit_record().return_once(|_| { + Some(Ok( + Box::new(MockRecord::new("key", "val")) as BoxedReadRecord + )) + }); + commiter.expect_flush().return_once(|| None).times(1); + + mock_ext + .expect_query() + .with(eq("bucket"), eq("entry"), predicate::always()) + .return_once(|_, _, _| Ok((processor, commiter))); + + let query = QueryEntry { + ext: Some(json!({ + "test1": {}, + "when": {"$limit": 1}, + })), + ..Default::default() + }; + + let mocked_ext_repo = mocked_ext_repo("test1", mock_ext); + + mocked_ext_repo + .register_query(1, "bucket", "entry", query) + .await + .unwrap(); + + let (tx, rx) = tokio::sync::mpsc::channel(2); + tx.send(Ok(record_reader)).await.unwrap(); + tx.send(Err(no_content!(""))).await.unwrap(); + + let query_rx = Arc::new(AsyncRwLock::new(rx)); + + assert!(mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .is_none()); + + mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .unwrap() + .expect("Should return a record"); + + assert!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .is_none(), + "Flush should not return any records" + ); + + assert_eq!( + mocked_ext_repo + .fetch_and_process_record(1, query_rx) + .await + .unwrap() + .err() + .unwrap(), + no_content!("") + ); + } + #[fixture] fn mock_ext() -> MockIoExtension { MockIoExtension::new() diff --git a/reductstore/src/storage/entry/io/record_reader.rs b/reductstore/src/storage/entry/io/record_reader.rs index 452615bf4..9a96b3447 100644 --- a/reductstore/src/storage/entry/io/record_reader.rs +++ b/reductstore/src/storage/entry/io/record_reader.rs @@ -8,7 +8,7 @@ use crate::storage::proto::Record; use crate::storage::storage::{CHANNEL_BUFFER_SIZE, IO_OPERATION_TIMEOUT, MAX_IO_BUFFER_SIZE}; use async_trait::async_trait; use bytes::Bytes; -use log::error; +use log::{debug, error}; use reduct_base::error::ReductError; use reduct_base::io::{ReadChunk, ReadRecord, RecordMeta}; use reduct_base::{internal_server_error, timeout}; @@ -167,9 +167,10 @@ impl RecordReader { }; if let Err(e) = read_all() { - error!( + // it's debug level because extensions may stop reading records intentionally + debug!( "Failed to send record {}/{}/{}: {}", - ctx.bucket_name, ctx.entry_name, ctx.record_timestamp, e + ctx.bucket_name, ctx.entry_name, ctx.record_timestamp, e.message ) } } From d3be67d9fd0a9c8429a326d38369a510e537ea77 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 12 Jun 2025 12:56:06 +0200 Subject: [PATCH 36/93] Filter buckets by read permission in server information (#849) * filter bucket by read permision * CHANGELOG * Update reductstore/src/api/entry/write_batched.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update reductstore/src/api/entry/update_single.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update reductstore/src/api/entry/update_batched.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update reductstore/src/api/entry/remove_single.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update reductstore/src/api/entry/remove_batched.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * add check to get endpoint * update CHANGELOG * fix api tests --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + integration_tests/api/bucket_api_test.py | 17 +++++- reductstore/src/api/bucket/create.rs | 2 +- reductstore/src/api/bucket/get.rs | 45 +++++++++++++- reductstore/src/api/bucket/head.rs | 2 +- reductstore/src/api/bucket/remove.rs | 2 +- reductstore/src/api/bucket/rename.rs | 2 +- reductstore/src/api/bucket/update.rs | 2 +- reductstore/src/api/entry/read_batched.rs | 2 +- reductstore/src/api/entry/read_query.rs | 2 +- reductstore/src/api/entry/read_query_post.rs | 2 +- reductstore/src/api/entry/read_single.rs | 2 +- reductstore/src/api/entry/remove_batched.rs | 2 +- reductstore/src/api/entry/remove_entry.rs | 2 +- reductstore/src/api/entry/remove_query.rs | 2 +- .../src/api/entry/remove_query_post.rs | 2 +- reductstore/src/api/entry/remove_single.rs | 2 +- reductstore/src/api/entry/rename_entry.rs | 2 +- reductstore/src/api/entry/update_batched.rs | 2 +- reductstore/src/api/entry/update_single.rs | 2 +- reductstore/src/api/entry/write_batched.rs | 2 +- reductstore/src/api/entry/write_single.rs | 2 +- reductstore/src/api/middleware.rs | 2 +- reductstore/src/api/replication/create.rs | 2 +- reductstore/src/api/replication/get.rs | 2 +- reductstore/src/api/replication/list.rs | 2 +- reductstore/src/api/replication/remove.rs | 2 +- reductstore/src/api/replication/update.rs | 2 +- reductstore/src/api/server/info.rs | 2 +- reductstore/src/api/server/list.rs | 59 +++++++++++++++++-- reductstore/src/api/token/create.rs | 2 +- reductstore/src/api/token/get.rs | 2 +- reductstore/src/api/token/list.rs | 2 +- reductstore/src/api/token/me.rs | 2 +- reductstore/src/api/token/remove.rs | 2 +- 35 files changed, 145 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e6f2ac0b..bf2665ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pass server information to extensions, [PR-816](https://github.com/reductstore/reductstore/pull/816) - Integrate ReductSelect v0.1.0, [PR-821](https://github.com/reductstore/reductstore/pull/821) +- Filter buckets by read permission in server information, [PR-849](https://github.com/reductstore/reductstore/pull/849) ### Changed diff --git a/integration_tests/api/bucket_api_test.py b/integration_tests/api/bucket_api_test.py index d6fbca72e..3e0676747 100644 --- a/integration_tests/api/bucket_api_test.py +++ b/integration_tests/api/bucket_api_test.py @@ -79,7 +79,12 @@ def test__create_bucket_custom(base_url, session, bucket_name): @requires_env("API_TOKEN") def test__get_bucket_with_authenticated_token( - base_url, session, bucket_name, token_without_permissions + base_url, + session, + bucket_name, + token_without_permissions, + token_read_bucket, + token_write_bucket, ): """Needs an authenticated token""" session.post(f"{base_url}/b/{bucket_name}") @@ -90,8 +95,18 @@ def test__get_bucket_with_authenticated_token( resp = session.get( f"{base_url}/b/{bucket_name}", headers=auth_headers(token_without_permissions) ) + assert resp.status_code == 403 + + resp = session.get( + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket) + ) assert resp.status_code == 200 + resp = session.get( + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket) + ) + assert resp.status_code == 403 + def test__get_bucket_stats(base_url, session, bucket_name): """Should get stats from bucket""" diff --git a/reductstore/src/api/bucket/create.rs b/reductstore/src/api/bucket/create.rs index 7794fea88..5c90595cb 100644 --- a/reductstore/src/api/bucket/create.rs +++ b/reductstore/src/api/bucket/create.rs @@ -17,7 +17,7 @@ pub(crate) async fn create_bucket( headers: HeaderMap, settings: BucketSettingsAxum, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; components .storage .create_bucket(&bucket_name, settings.into())?; diff --git a/reductstore/src/api/bucket/get.rs b/reductstore/src/api/bucket/get.rs index 6a2683bf9..b0b29bae9 100644 --- a/reductstore/src/api/bucket/get.rs +++ b/reductstore/src/api/bucket/get.rs @@ -5,7 +5,7 @@ use crate::api::bucket::FullBucketInfoAxum; use crate::api::middleware::check_permissions; use crate::api::Components; use crate::api::HttpError; -use crate::auth::policy::AuthenticatedPolicy; +use crate::auth::policy::ReadAccessPolicy; use axum::extract::{Path, State}; use axum_extra::headers::HeaderMap; use std::sync::Arc; @@ -17,7 +17,14 @@ pub(crate) async fn get_bucket( Path(bucket_name): Path, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, AuthenticatedPolicy {}).await?; + check_permissions( + &components, + &headers, + ReadAccessPolicy { + bucket: bucket_name.clone(), + }, + ) + .await?; let bucket_info = components.storage.get_bucket(&bucket_name)?.upgrade()?; Ok(bucket_info.info()?.into()) } @@ -32,7 +39,9 @@ mod tests { use rstest::rstest; + use axum::http::HeaderValue; use reduct_base::error::ErrorCode; + use reduct_base::msg::token_api::Permissions; use std::sync::Arc; #[rstest] @@ -64,4 +73,36 @@ mod tests { HttpError::new(ErrorCode::NotFound, "Bucket 'not-found' is not found") ) } + + #[rstest] + #[tokio::test] + async fn test_get_bucket_unauthorized( + #[future] components: Arc, + mut headers: HeaderMap, + ) { + let components = components.await; + let token = components + .token_repo + .write() + .await + .generate_token( + "test-token", + Permissions { + full_access: false, + read: vec!["bucket-1".to_string()], + write: vec![], + }, + ) + .unwrap(); + + headers.insert( + "Authorization", + HeaderValue::from_str(&format!("Bearer {}", token.value)).unwrap(), + ); + let err = get_bucket(State(components), Path("bucket-2".to_string()), headers) + .await + .err() + .unwrap(); + assert_eq!(err.0.status(), ErrorCode::Forbidden); + } } diff --git a/reductstore/src/api/bucket/head.rs b/reductstore/src/api/bucket/head.rs index f967ff368..3b359b11e 100644 --- a/reductstore/src/api/bucket/head.rs +++ b/reductstore/src/api/bucket/head.rs @@ -15,7 +15,7 @@ pub(crate) async fn head_bucket( Path(bucket_name): Path, headers: HeaderMap, ) -> Result<(), HttpError> { - check_permissions(&components, headers, AuthenticatedPolicy {}).await?; + check_permissions(&components, &headers, AuthenticatedPolicy {}).await?; components.storage.get_bucket(&bucket_name)?; Ok(()) } diff --git a/reductstore/src/api/bucket/remove.rs b/reductstore/src/api/bucket/remove.rs index b267e22db..3f74ccb64 100644 --- a/reductstore/src/api/bucket/remove.rs +++ b/reductstore/src/api/bucket/remove.rs @@ -16,7 +16,7 @@ pub(crate) async fn remove_bucket( Path(bucket_name): Path, headers: HeaderMap, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; components.storage.remove_bucket(&bucket_name).await?; components .token_repo diff --git a/reductstore/src/api/bucket/rename.rs b/reductstore/src/api/bucket/rename.rs index 68d39a358..b0b835bca 100644 --- a/reductstore/src/api/bucket/rename.rs +++ b/reductstore/src/api/bucket/rename.rs @@ -17,7 +17,7 @@ pub(crate) async fn rename_bucket( headers: HeaderMap, request: Json, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; components .storage .rename_bucket(&bucket_name, &request.new_name) diff --git a/reductstore/src/api/bucket/update.rs b/reductstore/src/api/bucket/update.rs index d745f8395..97c559725 100644 --- a/reductstore/src/api/bucket/update.rs +++ b/reductstore/src/api/bucket/update.rs @@ -16,7 +16,7 @@ pub(crate) async fn update_bucket( headers: HeaderMap, settings: BucketSettingsAxum, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; Ok(components .storage diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index 8cba89cf8..e2e788cc1 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -44,7 +44,7 @@ pub(crate) async fn read_batched_records( let entry_name = path.get("entry_name").unwrap(); check_permissions( &components, - headers, + &headers, ReadAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/read_query.rs b/reductstore/src/api/entry/read_query.rs index 8f2148638..08086a640 100644 --- a/reductstore/src/api/entry/read_query.rs +++ b/reductstore/src/api/entry/read_query.rs @@ -25,7 +25,7 @@ pub(crate) async fn read_query( check_permissions( &components, - headers, + &headers, ReadAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/read_query_post.rs b/reductstore/src/api/entry/read_query_post.rs index c493275f6..67c3365f0 100644 --- a/reductstore/src/api/entry/read_query_post.rs +++ b/reductstore/src/api/entry/read_query_post.rs @@ -24,7 +24,7 @@ pub(crate) async fn read_query_json( check_permissions( &components, - headers, + &headers, ReadAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/read_single.rs b/reductstore/src/api/entry/read_single.rs index 00a8766d2..e775f03ae 100644 --- a/reductstore/src/api/entry/read_single.rs +++ b/reductstore/src/api/entry/read_single.rs @@ -42,7 +42,7 @@ pub(crate) async fn read_record( let entry_name = path.get("entry_name").unwrap(); check_permissions( &components, - headers, + &headers, ReadAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/remove_batched.rs b/reductstore/src/api/entry/remove_batched.rs index ff1856c73..5bd40085e 100644 --- a/reductstore/src/api/entry/remove_batched.rs +++ b/reductstore/src/api/entry/remove_batched.rs @@ -25,7 +25,7 @@ pub(crate) async fn remove_batched_records( let bucket_name = path.get("bucket_name").unwrap(); check_permissions( &components, - headers.clone(), + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/remove_entry.rs b/reductstore/src/api/entry/remove_entry.rs index 7ad1bb754..a2a1ca49e 100644 --- a/reductstore/src/api/entry/remove_entry.rs +++ b/reductstore/src/api/entry/remove_entry.rs @@ -22,7 +22,7 @@ pub(crate) async fn remove_entry( check_permissions( &components, - headers, + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/remove_query.rs b/reductstore/src/api/entry/remove_query.rs index 5956b2590..b7ee36c34 100644 --- a/reductstore/src/api/entry/remove_query.rs +++ b/reductstore/src/api/entry/remove_query.rs @@ -27,7 +27,7 @@ pub(crate) async fn remove_query( check_permissions( &components, - headers, + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/remove_query_post.rs b/reductstore/src/api/entry/remove_query_post.rs index 97409cbb1..4679c233e 100644 --- a/reductstore/src/api/entry/remove_query_post.rs +++ b/reductstore/src/api/entry/remove_query_post.rs @@ -32,7 +32,7 @@ pub(crate) async fn remove_query_json( check_permissions( &components, - headers, + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/remove_single.rs b/reductstore/src/api/entry/remove_single.rs index a4655fb21..ba2ce152f 100644 --- a/reductstore/src/api/entry/remove_single.rs +++ b/reductstore/src/api/entry/remove_single.rs @@ -22,7 +22,7 @@ pub(crate) async fn remove_record( let bucket = path.get("bucket_name").unwrap(); check_permissions( &components, - headers.clone(), + &headers, WriteAccessPolicy { bucket: bucket.clone(), }, diff --git a/reductstore/src/api/entry/rename_entry.rs b/reductstore/src/api/entry/rename_entry.rs index 7274fe76a..169f5bd04 100644 --- a/reductstore/src/api/entry/rename_entry.rs +++ b/reductstore/src/api/entry/rename_entry.rs @@ -23,7 +23,7 @@ pub(crate) async fn rename_entry( check_permissions( &components, - headers, + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/update_batched.rs b/reductstore/src/api/entry/update_batched.rs index 7a7963678..173d3bb37 100644 --- a/reductstore/src/api/entry/update_batched.rs +++ b/reductstore/src/api/entry/update_batched.rs @@ -29,7 +29,7 @@ pub(crate) async fn update_batched_records( let bucket_name = path.get("bucket_name").unwrap(); check_permissions( &components, - headers.clone(), + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/update_single.rs b/reductstore/src/api/entry/update_single.rs index 6b9a68a17..79cda488e 100644 --- a/reductstore/src/api/entry/update_single.rs +++ b/reductstore/src/api/entry/update_single.rs @@ -28,7 +28,7 @@ pub(crate) async fn update_record( let bucket = path.get("bucket_name").unwrap(); check_permissions( &components, - headers.clone(), + &headers, WriteAccessPolicy { bucket: bucket.clone(), }, diff --git a/reductstore/src/api/entry/write_batched.rs b/reductstore/src/api/entry/write_batched.rs index 2454db2d7..e617e8d19 100644 --- a/reductstore/src/api/entry/write_batched.rs +++ b/reductstore/src/api/entry/write_batched.rs @@ -45,7 +45,7 @@ pub(crate) async fn write_batched_records( let bucket_name = path.get("bucket_name").unwrap().clone(); check_permissions( &components, - headers.clone(), + &headers, WriteAccessPolicy { bucket: bucket_name.clone(), }, diff --git a/reductstore/src/api/entry/write_single.rs b/reductstore/src/api/entry/write_single.rs index 67d4c1a84..188ef1ace 100644 --- a/reductstore/src/api/entry/write_single.rs +++ b/reductstore/src/api/entry/write_single.rs @@ -31,7 +31,7 @@ pub(crate) async fn write_record( let bucket = path.get("bucket_name").unwrap(); check_permissions( &components, - headers.clone(), + &headers.clone(), WriteAccessPolicy { bucket: bucket.clone(), }, diff --git a/reductstore/src/api/middleware.rs b/reductstore/src/api/middleware.rs index 36e80eb8b..742d03354 100644 --- a/reductstore/src/api/middleware.rs +++ b/reductstore/src/api/middleware.rs @@ -63,7 +63,7 @@ pub async fn print_statuses( pub(crate) async fn check_permissions

( components: &Components, - headers: HeaderMap, + headers: &HeaderMap, policy: P, ) -> Result<(), HttpError> where diff --git a/reductstore/src/api/replication/create.rs b/reductstore/src/api/replication/create.rs index 49ad4a71d..62daa6ab4 100644 --- a/reductstore/src/api/replication/create.rs +++ b/reductstore/src/api/replication/create.rs @@ -16,7 +16,7 @@ pub(crate) async fn create_replication( headers: HeaderMap, settings: ReplicationSettingsAxum, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; components .replication_repo diff --git a/reductstore/src/api/replication/get.rs b/reductstore/src/api/replication/get.rs index 4368e96e5..6f8756f73 100644 --- a/reductstore/src/api/replication/get.rs +++ b/reductstore/src/api/replication/get.rs @@ -15,7 +15,7 @@ pub(crate) async fn get_replication( Path(replication_name): Path, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; let info = components .replication_repo diff --git a/reductstore/src/api/replication/list.rs b/reductstore/src/api/replication/list.rs index 005a38d5f..6f7c70966 100644 --- a/reductstore/src/api/replication/list.rs +++ b/reductstore/src/api/replication/list.rs @@ -16,7 +16,7 @@ pub(crate) async fn list_replications( State(components): State>, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; let mut list = ReplicationListAxum::default(); diff --git a/reductstore/src/api/replication/remove.rs b/reductstore/src/api/replication/remove.rs index 8cc801675..29177b65a 100644 --- a/reductstore/src/api/replication/remove.rs +++ b/reductstore/src/api/replication/remove.rs @@ -15,7 +15,7 @@ pub(crate) async fn remove_replication( Path(replication_name): Path, headers: HeaderMap, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; components .replication_repo diff --git a/reductstore/src/api/replication/update.rs b/reductstore/src/api/replication/update.rs index e91f9d3ab..ce4f46e8d 100644 --- a/reductstore/src/api/replication/update.rs +++ b/reductstore/src/api/replication/update.rs @@ -16,7 +16,7 @@ pub(crate) async fn update_replication( headers: HeaderMap, settings: ReplicationSettingsAxum, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; components .replication_repo diff --git a/reductstore/src/api/server/info.rs b/reductstore/src/api/server/info.rs index 7d247ce4a..65da9a94b 100644 --- a/reductstore/src/api/server/info.rs +++ b/reductstore/src/api/server/info.rs @@ -15,7 +15,7 @@ pub(crate) async fn info( State(components): State>, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, AuthenticatedPolicy {}).await?; + check_permissions(&components, &headers, AuthenticatedPolicy {}).await?; Ok(components.storage.info()?.into()) } diff --git a/reductstore/src/api/server/list.rs b/reductstore/src/api/server/list.rs index 4e33e441d..5ce15eb37 100644 --- a/reductstore/src/api/server/list.rs +++ b/reductstore/src/api/server/list.rs @@ -1,12 +1,13 @@ -// Copyright 2023-2024 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 use crate::api::middleware::check_permissions; use crate::api::server::BucketInfoListAxum; use crate::api::{Components, HttpError}; -use crate::auth::policy::AuthenticatedPolicy; +use crate::auth::policy::{AuthenticatedPolicy, ReadAccessPolicy}; use axum::extract::State; use axum_extra::headers::HeaderMap; +use reduct_base::msg::server_api::BucketInfoList; use std::sync::Arc; // GET /list @@ -14,16 +15,38 @@ pub(crate) async fn list( State(components): State>, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, AuthenticatedPolicy {}).await?; + check_permissions(&components, &headers, AuthenticatedPolicy {}).await?; - let list = components.storage.get_bucket_list()?; - Ok(list.into()) + let mut filtered_by_read_permission = vec![]; + + for bucket in components.storage.get_bucket_list()?.buckets { + // Filter out buckets that are not visible to the user + if check_permissions( + &components, + &headers, + ReadAccessPolicy { + bucket: bucket.name.clone(), + }, + ) + .await + .is_ok() + { + filtered_by_read_permission.push(bucket); + } + } + + Ok(BucketInfoList { + buckets: filtered_by_read_permission, + } + .into()) } #[cfg(test)] mod tests { use super::*; use crate::api::tests::{components, headers}; + use axum::http::HeaderValue; + use reduct_base::msg::token_api::Permissions; use rstest::rstest; #[rstest] @@ -32,4 +55,30 @@ mod tests { let list = list(State(components.await), headers).await.unwrap(); assert_eq!(list.0.buckets.len(), 2); } + + #[rstest] + #[tokio::test] + async fn test_filtered_list(#[future] components: Arc, mut headers: HeaderMap) { + let components = components.await; + let token = components + .token_repo + .write() + .await + .generate_token( + "with-one-bucket", + Permissions { + read: vec!["bucket-1".to_string()], + ..Default::default() + }, + ) + .unwrap(); + + headers.insert( + "Authorization", + HeaderValue::from_str(&format!("Bearer {}", token.value)).unwrap(), + ); + let list = list(State(components), headers).await.unwrap(); + assert_eq!(list.0.buckets.len(), 1); + assert_eq!(list.0.buckets[0].name, "bucket-1"); + } } diff --git a/reductstore/src/api/token/create.rs b/reductstore/src/api/token/create.rs index e3271527d..ac214defe 100644 --- a/reductstore/src/api/token/create.rs +++ b/reductstore/src/api/token/create.rs @@ -16,7 +16,7 @@ pub(crate) async fn create_token( headers: HeaderMap, permissions: PermissionsAxum, ) -> Result { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; Ok(TokenCreateResponseAxum( components diff --git a/reductstore/src/api/token/get.rs b/reductstore/src/api/token/get.rs index 965a96c01..e77bd246e 100644 --- a/reductstore/src/api/token/get.rs +++ b/reductstore/src/api/token/get.rs @@ -15,7 +15,7 @@ pub(crate) async fn get_token( Path(token_name): Path, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; let mut token = components .token_repo diff --git a/reductstore/src/api/token/list.rs b/reductstore/src/api/token/list.rs index 9a106e74c..2c69f3f88 100644 --- a/reductstore/src/api/token/list.rs +++ b/reductstore/src/api/token/list.rs @@ -15,7 +15,7 @@ pub(crate) async fn list_tokens( State(components): State>, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; let token_repo = components.token_repo.read().await; let mut list = TokenListAxum::default(); diff --git a/reductstore/src/api/token/me.rs b/reductstore/src/api/token/me.rs index cab9d098a..6b50876ab 100644 --- a/reductstore/src/api/token/me.rs +++ b/reductstore/src/api/token/me.rs @@ -15,7 +15,7 @@ pub(crate) async fn me( State(components): State>, headers: HeaderMap, ) -> Result { - check_permissions(&components, headers.clone(), AuthenticatedPolicy {}).await?; + check_permissions(&components, &headers.clone(), AuthenticatedPolicy {}).await?; let header = match headers.get("Authorization") { Some(header) => header.to_str().ok(), None => None, diff --git a/reductstore/src/api/token/remove.rs b/reductstore/src/api/token/remove.rs index 581b235fa..f488f357f 100644 --- a/reductstore/src/api/token/remove.rs +++ b/reductstore/src/api/token/remove.rs @@ -14,7 +14,7 @@ pub(crate) async fn remove_token( Path(token_name): Path, headers: HeaderMap, ) -> Result<(), HttpError> { - check_permissions(&components, headers, FullAccessPolicy {}).await?; + check_permissions(&components, &headers, FullAccessPolicy {}).await?; Ok(components .token_repo From 064ec235c7d6d91e9507c28d354d7a8a62f82293 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 13 Jun 2025 10:47:28 +0200 Subject: [PATCH 37/93] squash commit (#850) --- CHANGELOG.md | 1 + reductstore/build.rs | 16 ++++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf2665ac4..1a7c56fc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) - Run all operands after a compute-staged one on the compute stage, [PR-835](https://github.com/reductstore/reductstore/pull/835) - Replace auto-staging for extension filtering with when condition in ext parameter, [PR-838](https://github.com/reductstore/reductstore/pull/838) +- Update ReductSelect to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) ### Fixed diff --git a/reductstore/build.rs b/reductstore/build.rs index c8071c52d..5f6fd6baf 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -30,7 +30,7 @@ fn main() -> Result<(), Box> { download_web_console("v1.10.2"); #[cfg(feature = "select-ext")] - download_ext("select-ext", "v0.2.0"); + download_ext("select-ext", "v0.3.0"); // get build time and commit let build_time = chrono::DateTime::::from(SystemTime::now()) @@ -74,12 +74,8 @@ fn download_web_console(version: &str) { #[cfg(feature = "select-ext")] fn download_ext(name: &str, version: &str) { - let sas_url = env::var("ARTIFACT_SAS_URL").unwrap_or_default(); - if sas_url.is_empty() { - panic!("ARTIFACT_SAS_URL is not set, disable the extensions feature"); - } - - let sas_url = Url::parse(&sas_url).expect("Failed to parse ARTIFACT_SAS_URL"); + let artifacts_host_url = + Url::parse("https://reductsoft.z6.web.core.windows.net/").expect("Failed to parse SAS URL"); let target = env::var("TARGET").unwrap(); let out_dir = env::var("OUT_DIR").unwrap(); @@ -90,13 +86,13 @@ fn download_ext(name: &str, version: &str) { } println!("Downloading {}...", name); - let mut ext_url = sas_url + let mut ext_url = artifacts_host_url .join(&format!( - "/artifacts/{}/{}/{}.zip/{}.zip", + "/{}/{}/{}.zip/{}.zip", name, version, target, target )) .expect("Failed to create URL"); - ext_url.set_query(sas_url.query()); + ext_url.set_query(artifacts_host_url.query()); let client = Client::builder().user_agent("ReductStore").build().unwrap(); let resp = client From 223b29140b180132bb23e2afc96179b1d5ba0354 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 13:43:17 +0200 Subject: [PATCH 38/93] Bump syn from 2.0.101 to 2.0.103 (#853) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.101 to 2.0.103. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.101...2.0.103) --- updated-dependencies: - dependency-name: syn dependency-version: 2.0.103 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 62 ++++++++++++++++++++-------------------- reduct_macros/Cargo.toml | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0fedf8bc1..788a830f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,7 +148,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -159,7 +159,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -283,7 +283,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -354,7 +354,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.101", + "syn 2.0.103", "which", ] @@ -583,7 +583,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -605,7 +605,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -628,7 +628,7 @@ checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -812,7 +812,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1227,7 +1227,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1563,7 +1563,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1772,7 +1772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ "proc-macro2", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -1829,7 +1829,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.101", + "syn 2.0.103", "tempfile", ] @@ -1843,7 +1843,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2032,7 +2032,7 @@ name = "reduct-macros" version = "1.16.0" dependencies = [ "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2217,7 +2217,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.101", + "syn 2.0.103", "unicode-ident", ] @@ -2376,7 +2376,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2436,7 +2436,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2542,9 +2542,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" dependencies = [ "proc-macro2", "quote", @@ -2568,7 +2568,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2609,7 +2609,7 @@ checksum = "888d0c3c6db53c0fdab160d2ed5e12ba745383d3e85813f2ea0f2b1475ab553f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2629,7 +2629,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2734,7 +2734,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -2922,7 +2922,7 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -3031,7 +3031,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "wasm-bindgen-shared", ] @@ -3066,7 +3066,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3321,7 +3321,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "synstructure", ] @@ -3342,7 +3342,7 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -3362,7 +3362,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", "synstructure", ] @@ -3383,7 +3383,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] @@ -3405,7 +3405,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.103", ] [[package]] diff --git a/reduct_macros/Cargo.toml b/reduct_macros/Cargo.toml index 47dbdca3f..1d4d10450 100644 --- a/reduct_macros/Cargo.toml +++ b/reduct_macros/Cargo.toml @@ -15,5 +15,5 @@ keywords = ["database", "time-series", "blob", "storage", "reductstore"] proc-macro = true [dependencies] -syn = { version = "2.0.101", features = ["derive"] } +syn = { version = "2.0.103", features = ["derive"] } quote = "1.0.40" From 97d0ee3edb87d3520b305ec5c395607eb9d1644e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 13:43:30 +0200 Subject: [PATCH 39/93] Bump zip from 4.0.0 to 4.1.0 (#852) Bumps [zip](https://github.com/zip-rs/zip2) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/zip-rs/zip2/releases) - [Changelog](https://github.com/zip-rs/zip2/blob/master/CHANGELOG.md) - [Commits](https://github.com/zip-rs/zip2/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: zip dependency-version: 4.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 788a830f8..c07a2639a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3410,9 +3410,9 @@ dependencies = [ [[package]] name = "zip" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "153a6fff49d264c4babdcfa6b4d534747f520e56e8f0f384f3b808c4b64cc1fd" +checksum = "af7dcdb4229c0e79c2531a24de7726a0e980417a74fb4d030a35f535665439a0" dependencies = [ "aes", "arbitrary", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index fe47a2ada..a63e25efc 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -31,7 +31,7 @@ reduct-base = { path = "../reduct_base", version = "1.15.0", features = ["ext"] reduct-macros = { path = "../reduct_macros", version = "1.15.0" } chrono = { version = "0.4.41", features = ["serde"] } -zip = "4.0.0" +zip = "4.1.0" tempfile = "3.20.0" hex = "0.4.3" prost-wkt-types = "0.6.1" From 9a411547ebcc178147120630482cb1bf845fef43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 13:43:42 +0200 Subject: [PATCH 40/93] Bump reqwest from 0.12.19 to 0.12.20 (#854) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.19 to 0.12.20. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.19...v0.12.20) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.12.20 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 7 ++----- reductstore/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c07a2639a..73290c9db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2134,9 +2134,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.19" +version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" +checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ "base64 0.22.1", "bytes", @@ -2149,11 +2149,8 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "ipnet", "js-sys", "log", - "mime", - "once_cell", "percent-encoding", "pin-project-lite", "quinn", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index a63e25efc..da6491a1c 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -66,7 +66,7 @@ prost = "0.13.1" [build-dependencies] prost-build = "0.13.1" -reqwest = { version = "0.12.19", default-features = false, features = ["rustls-tls", "blocking", "json"] } +reqwest = { version = "0.12.20", default-features = false, features = ["rustls-tls", "blocking", "json"] } chrono = "0.4.41" serde_json = "1.0.140" @@ -75,7 +75,7 @@ mockall = "0.13.1" rstest = "0.25.0" serial_test = "3.2.0" test-log = "0.2.17" -reqwest = { version = "0.12.19", default-features = false, features = ["rustls-tls", "blocking"] } +reqwest = { version = "0.12.20", default-features = false, features = ["rustls-tls", "blocking"] } assert_matches = "1.5" [package.metadata.docs.rs] From 6acc6b86af69fecf5c0797ec59d2264a22537edb Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 19 Jun 2025 19:12:48 +0200 Subject: [PATCH 41/93] Integrate ReductRos v0.1.0 (#856) * integrate ros-ext * update CHANGELOG * fix path * add test for unparsable header --- CHANGELOG.md | 1 + integration_tests/api/data/file.mcap | Bin 0 -> 659 bytes integration_tests/api/ext_test.py | 42 +++++++++++++++++++++++++++ reductstore/Cargo.toml | 1 + reductstore/build.rs | 5 +++- reductstore/src/api.rs | 25 ++++++++++++++-- reductstore/src/cfg.rs | 15 +++++++++- 7 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 integration_tests/api/data/file.mcap diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a7c56fc2..86d7e4900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Pass server information to extensions, [PR-816](https://github.com/reductstore/reductstore/pull/816) - Integrate ReductSelect v0.1.0, [PR-821](https://github.com/reductstore/reductstore/pull/821) +- Integrate ReductRos v0.1.0, [PR-856](https://github.com/reductstore/reductstore/pull/856) - Filter buckets by read permission in server information, [PR-849](https://github.com/reductstore/reductstore/pull/849) ### Changed diff --git a/integration_tests/api/data/file.mcap b/integration_tests/api/data/file.mcap new file mode 100644 index 0000000000000000000000000000000000000000..87271d679c7e7b05d1831d007cfb950161c457ec GIT binary patch literal 659 zcmeD5b#@Fe;N@ZzV?Y2tATu{Pu|T(|Sl2+$$VktCZ7Eci4N8kc>FrSZ;ui5MEI@Wu zaY;%gR7_*5{$GZOR%V8vVxQZKG&_=vTMz+%gYNiR>w5)PG$-zyD+VuL0ZjxVZh9wkYzir)_;&} zWa462z&#;z#g2WJ8^T)R*R`!M+PCWFB0b3lX?8WJ3m6#`fGn7sW1*rjw|7DL{2+&e zfS~^bE@op0n}Lx*7{~&8CqB10y;vVe=m(b+W#*-`1BHw7i;aLH++Y zxL!sERv=ryB(=DN8OTUZDFR7xTYyL)V1iIUOBkUn5Cvkx91C+K7g!vop2H8M4hYbd zsY8Sr)= 0 + resp = session.get(f"{base_url}/b/{bucket}/entry/batch?q={query_id}") + assert resp.status_code == 200 + + assert ( + resp.headers["x-reduct-time-24"] + == "16,application/json,encoding=cdr,schema=std_msgs/String,topic=/test" + ) + assert resp.content == b'{"data":"hello"}' diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index da6491a1c..ce5c8721e 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -20,6 +20,7 @@ include = ["src/**/*", "Cargo.toml", "Cargo.lock", "build.rs", "README.md", "LIC default = ["web-console"] web-console = [] select-ext = [] +ros-ext = [] [lib] crate-type = ["lib"] diff --git a/reductstore/build.rs b/reductstore/build.rs index 5f6fd6baf..eea1aa4dd 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -32,6 +32,9 @@ fn main() -> Result<(), Box> { #[cfg(feature = "select-ext")] download_ext("select-ext", "v0.3.0"); + #[cfg(feature = "ros-ext")] + download_ext("ros-ext", "v0.1.0"); + // get build time and commit let build_time = chrono::DateTime::::from(SystemTime::now()) .to_rfc3339_opts(chrono::SecondsFormat::Secs, true); @@ -72,7 +75,7 @@ fn download_web_console(version: &str) { fs::copy(console_path, format!("{}/console.zip", out_dir)).expect("Failed to copy console.zip"); } -#[cfg(feature = "select-ext")] +#[allow(dead_code)] fn download_ext(name: &str, version: &str) { let artifacts_host_url = Url::parse("https://reductsoft.z6.web.core.windows.net/").expect("Failed to parse SAS URL"); diff --git a/reductstore/src/api.rs b/reductstore/src/api.rs index 9fa1fa52b..87d7e3246 100644 --- a/reductstore/src/api.rs +++ b/reductstore/src/api.rs @@ -26,7 +26,7 @@ use axum::{middleware::from_fn, Router}; use bucket::create_bucket_api_routes; use entry::create_entry_api_routes; use hyper::http::HeaderValue; -use log::warn; +use log::{error, warn}; use middleware::{default_headers, print_statuses}; pub use reduct_base::error::ErrorCode; use reduct_base::error::ReductError as BaseHttpError; @@ -92,11 +92,16 @@ impl IntoResponse for HttpError { StatusCode::from_u16(err.status as u16).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR) }; + let err_msg = if let Ok(header_value) = HeaderValue::from_str(&err.message) { + header_value + } else { + error!("Invalid error message: {}", err.message); + HeaderValue::from_str("Unparsable message").unwrap() + }; let mut resp = (http_code, body).into_response(); resp.headers_mut() .insert("content-type", "application/json".parse().unwrap()); - resp.headers_mut() - .insert("x-reduct-error", err.message.parse().unwrap()); + resp.headers_mut().insert("x-reduct-error", err_msg); resp } } @@ -243,6 +248,20 @@ mod tests { let body: Bytes = to_bytes(resp.into_body(), 1000).await.unwrap(); assert_eq!(body, Bytes::from(r#"{"detail": "Test 'error'"}"#)) } + + #[rstest] + #[tokio::test] + async fn test_http_error_unparsable_message() { + let error = HttpError::new( + ErrorCode::BadRequest, + &String::from_utf8_lossy(b"Test \x7f"), + ); + let resp = error.into_response(); + assert_eq!( + resp.headers().get("x-reduct-error").unwrap(), + HeaderValue::from_static("Unparsable message") + ); + } } #[fixture] diff --git a/reductstore/src/cfg.rs b/reductstore/src/cfg.rs index 82e3f049e..38f96c1b1 100644 --- a/reductstore/src/cfg.rs +++ b/reductstore/src/cfg.rs @@ -95,6 +95,7 @@ impl Cfg { let token_repo = self.provision_tokens(); let console = create_asset_manager(load_console()); let select_ext = create_asset_manager(load_select_ext()); + let ros_ext = create_asset_manager(load_ros_ext()); let replication_engine = self.provision_replication_repo(Arc::clone(&storage))?; let ext_path = if let Some(ext_path) = &self.ext_path { @@ -115,7 +116,7 @@ impl Cfg { replication_repo: tokio::sync::RwLock::new(replication_engine), ext_repo: create_ext_repository( ext_path, - vec![select_ext], + vec![select_ext, ros_ext], ExtSettings::builder() .log_level(&self.log_level) .server_info(server_info) @@ -162,6 +163,18 @@ fn load_select_ext() -> &'static [u8] { b"" } +#[cfg(feature = "ros-ext")] +fn load_ros_ext() -> &'static [u8] { + info!("Load Reduct ROS Extension"); + include_bytes!(concat!(env!("OUT_DIR"), "/ros-ext.zip")) +} + +#[cfg(not(feature = "ros-ext"))] +fn load_ros_ext() -> &'static [u8] { + info!("Reduct ROS Extension is disabled"); + b"" +} + impl Display for Cfg { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.env.message()) From 4623b0072e848c53d88503008331e6ded9510f51 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Mon, 23 Jun 2025 10:01:43 +0200 Subject: [PATCH 42/93] update deps in bulk --- Cargo.lock | 738 +++++++++++++++++++++++++++++------------------------ 1 file changed, 409 insertions(+), 329 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73290c9db..668bf03bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aes" @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -69,44 +69,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", - "once_cell", + "once_cell_polyfill", "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arbitrary" @@ -148,7 +148,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -159,7 +159,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -170,15 +170,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.12.6" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" dependencies = [ "aws-lc-sys", "zeroize", @@ -186,9 +186,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.27.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" dependencies = [ "bindgen", "cc", @@ -283,7 +283,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -310,9 +310,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -320,15 +320,9 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets", + "windows-targets 0.52.6", ] -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -354,15 +348,15 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.103", + "syn 2.0.104", "which", ] [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "block-buffer" @@ -375,9 +369,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" [[package]] name = "byteorder" @@ -418,9 +412,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.17" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "jobserver", "libc", @@ -438,9 +432,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -495,9 +489,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "constant_time_eq" @@ -568,9 +562,9 @@ checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] @@ -583,7 +577,7 @@ checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -605,7 +599,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -622,13 +616,13 @@ dependencies = [ [[package]] name = "dlopen2_derive" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b99bf03862d7f545ebc28ddd33a665b50865f4dfd84031a393823879bd4c54" +checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -660,9 +654,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -688,12 +682,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -710,9 +704,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -742,9 +736,9 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" [[package]] name = "fs-err" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa" +checksum = "88d7be93788013f265201256d58f04936a8079ad5dc898743aa20525f503b683" dependencies = [ "autocfg", "tokio", @@ -812,7 +806,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -863,22 +857,22 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "js-sys", @@ -902,9 +896,9 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -921,17 +915,17 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" [[package]] name = "headers" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64", "bytes", "headers-core", "http", @@ -1048,11 +1042,10 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.5" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "futures-util", "http", "hyper", "hyper-util", @@ -1061,16 +1054,16 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 0.26.8", + "webpki-roots", ] [[package]] name = "hyper-util" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", @@ -1090,9 +1083,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.62" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2fd658b06e56721792c5df4475705b6cda790e9298d19d2f8af083457bcd127" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1114,21 +1107,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -1137,31 +1131,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -1169,67 +1143,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.103", -] - [[package]] name = "idna" version = "1.0.3" @@ -1243,9 +1204,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -1253,9 +1214,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown", @@ -1348,10 +1309,11 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.3", "libc", ] @@ -1371,7 +1333,7 @@ version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64 0.22.1", + "base64", "js-sys", "pem", "ring", @@ -1394,34 +1356,34 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets", + "windows-targets 0.53.2", ] [[package]] name = "liblzma" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66352d7a8ac12d4877b6e6ea5a9b7650ee094257dc40889955bea5bc5b08c1d0" +checksum = "0791ab7e08ccc8e0ce893f6906eb2703ed8739d8e89b57c0714e71bad09024c8" dependencies = [ "liblzma-sys", ] [[package]] name = "liblzma-sys" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5839bad90c3cc2e0b8c4ed8296b80e86040240f81d46b9c0e9bc8dd51ddd3af1" +checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" dependencies = [ "cc", "libc", @@ -1430,9 +1392,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -1445,38 +1407,38 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", ] -[[package]] -name = "lockfree-object-pool" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" - [[package]] name = "log" version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.1.0" @@ -1494,9 +1456,9 @@ checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "mime" @@ -1522,22 +1484,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -1563,14 +1525,14 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] name = "multimap" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "nom" @@ -1641,6 +1603,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "overload" version = "0.1.1" @@ -1649,9 +1617,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -1659,15 +1627,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1686,7 +1654,7 @@ version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" dependencies = [ - "base64 0.22.1", + "base64", "serde", ] @@ -1724,6 +1692,15 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1767,12 +1744,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" dependencies = [ "proc-macro2", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -1791,14 +1768,14 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ - "toml_edit 0.22.24", + "toml_edit 0.22.27", ] [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -1829,7 +1806,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.103", + "syn 2.0.104", "tempfile", ] @@ -1843,7 +1820,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -1903,9 +1880,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", "cfg_aliases", @@ -1923,12 +1900,13 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", - "getrandom 0.3.2", + "getrandom 0.3.3", + "lru-slab", "rand", "ring", "rustc-hash 2.1.1", @@ -1943,9 +1921,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" dependencies = [ "cfg_aliases", "libc", @@ -1966,9 +1944,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" @@ -1996,14 +1974,14 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", ] [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ "bitflags", ] @@ -2032,7 +2010,7 @@ name = "reduct-macros" version = "1.16.0" dependencies = [ "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2045,7 +2023,7 @@ dependencies = [ "axum", "axum-extra", "axum-server", - "base64 0.22.1", + "base64", "byteorder", "bytes", "bytesize", @@ -2138,7 +2116,7 @@ version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-core", @@ -2171,7 +2149,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.0", + "webpki-roots", ] [[package]] @@ -2182,7 +2160,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -2214,15 +2192,15 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.103", + "syn 2.0.104", "unicode-ident", ] [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -2260,22 +2238,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys 0.9.3", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.27" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "aws-lc-rs", "log", @@ -2298,11 +2276,12 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time", + "zeroize", ] [[package]] @@ -2319,9 +2298,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "ryu" @@ -2331,9 +2310,9 @@ checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "scc" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea091f6cac2595aa38993f04f4ee692ed43757035c36e67c180b6828356385b1" +checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4" dependencies = [ "sdd", ] @@ -2373,7 +2352,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2433,7 +2412,7 @@ checksum = "5d69265a08751de7844521fd15003ae0a888e035773ba05695c5c759a6f89eef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2464,9 +2443,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -2491,18 +2470,15 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" @@ -2539,9 +2515,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.103" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -2559,13 +2535,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2575,9 +2551,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.2", + "getrandom 0.3.3", "once_cell", - "rustix 1.0.3", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -2606,7 +2582,7 @@ checksum = "888d0c3c6db53c0fdab160d2ed5e12ba745383d3e85813f2ea0f2b1475ab553f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2626,7 +2602,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2641,12 +2617,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -2682,9 +2657,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -2731,7 +2706,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2746,9 +2721,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -2759,9 +2734,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" [[package]] name = "toml_edit" @@ -2776,13 +2751,13 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "toml_datetime", - "winnow 0.7.4", + "winnow 0.7.11", ] [[package]] @@ -2844,9 +2819,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -2919,7 +2894,7 @@ checksum = "35f5380909ffc31b4de4f4bdf96b877175a016aa2ca98cee39fcfd8c4d53d952" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -2952,12 +2927,6 @@ dependencies = [ "serde", ] -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -2993,9 +2962,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -3028,7 +2997,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -3063,7 +3032,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3112,18 +3081,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "webpki-roots" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2853738d1cc4f2da3a225c18ec6c3721abb31961096e9dbf5ab35fa88b19cfdb" +checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" dependencies = [ "rustls-pki-types", ] @@ -3164,18 +3124,62 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.52.0" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ - "windows-targets", + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", ] [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-result" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] [[package]] name = "windows-sys" @@ -3183,7 +3187,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -3192,7 +3196,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", ] [[package]] @@ -3201,14 +3214,30 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", ] [[package]] @@ -3217,48 +3246,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.5.40" @@ -3270,9 +3347,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.4" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ "memchr", ] @@ -3286,23 +3363,17 @@ dependencies = [ "bitflags", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -3312,34 +3383,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] @@ -3359,7 +3430,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", "synstructure", ] @@ -3380,14 +3451,25 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", ] [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -3396,20 +3478,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.103", + "syn 2.0.104", ] [[package]] name = "zip" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7dcdb4229c0e79c2531a24de7726a0e980417a74fb4d030a35f535665439a0" +checksum = "95ab361742de920c5535880f89bbd611ee62002bf11341d16a5f057bb8ba6899" dependencies = [ "aes", "arbitrary", @@ -3418,7 +3500,7 @@ dependencies = [ "crc32fast", "deflate64", "flate2", - "getrandom 0.3.2", + "getrandom 0.3.3", "hmac", "indexmap", "liblzma", @@ -3433,21 +3515,19 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zopfli" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" dependencies = [ "bumpalo", "crc32fast", - "lockfree-object-pool", "log", - "once_cell", "simd-adler32", ] From 0e677220901c9620c686d310282597438db6edc4 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 24 Jun 2025 13:58:35 +0200 Subject: [PATCH 43/93] Support for duration literals (#864) * add duration literal * update CHANGELOG * duration is microsecond; * improve tests * add comments --- CHANGELOG.md | 1 + .../condition/operators/aggregation/each_t.rs | 13 +- .../query/condition/operators/misc/cast.rs | 2 +- .../query/condition/operators/misc/exists.rs | 2 +- .../query/condition/operators/misc/ref.rs | 2 +- .../src/storage/query/condition/parser.rs | 18 +- .../src/storage/query/condition/value.rs | 76 +++++--- .../query/condition/value/arithmetic/abs.rs | 3 +- .../query/condition/value/arithmetic/add.rs | 17 +- .../query/condition/value/arithmetic/div.rs | 5 + .../condition/value/arithmetic/div_num.rs | 6 + .../query/condition/value/arithmetic/mult.rs | 19 +- .../query/condition/value/arithmetic/rem.rs | 19 +- .../query/condition/value/arithmetic/sub.rs | 21 ++- .../src/storage/query/condition/value/cmp.rs | 57 +++++- .../query/condition/value/duration_format.rs | 177 ++++++++++++++++++ .../query/condition/value/misc/cast.rs | 12 +- .../query/condition/value/string/contains.rs | 4 +- .../query/condition/value/string/ends_with.rs | 4 +- .../condition/value/string/starts_with.rs | 4 +- 20 files changed, 386 insertions(+), 76 deletions(-) create mode 100644 reductstore/src/storage/query/condition/value/duration_format.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 86d7e4900..682442387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Integrate ReductSelect v0.1.0, [PR-821](https://github.com/reductstore/reductstore/pull/821) - Integrate ReductRos v0.1.0, [PR-856](https://github.com/reductstore/reductstore/pull/856) - Filter buckets by read permission in server information, [PR-849](https://github.com/reductstore/reductstore/pull/849) +- Support for duration literals, [PR-864](https://github.com/reductstore/reductstore/pull/864) ### Changed diff --git a/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs b/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs index 9a89af79a..60e9271a6 100644 --- a/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs +++ b/reductstore/src/storage/query/condition/operators/aggregation/each_t.rs @@ -39,7 +39,12 @@ impl Node for EachT { } let last_time = self.last_timestamp.unwrap(); - let s = self.operands[0].apply(context)?.as_float()?; + let value = self.operands[0].apply(context)?; + let s = if value.is_duration() { + value.as_float()? / 1_000_000.0 + } else { + value.as_float()? + }; let ret = context.timestamp - last_time >= (s * 1_000_000.0) as u64; if ret { @@ -63,8 +68,10 @@ mod tests { use rstest::rstest; #[rstest] - fn apply_ok() { - let mut op = EachT::new(vec![Constant::boxed(Value::Float(0.1))]); + #[case(Value::Float(0.1))] + #[case(Value::Duration(100_000))] + fn apply_ok(#[case] value: Value) { + let mut op = EachT::new(vec![Constant::boxed(value)]); let mut context = Context::default(); assert_eq!(op.apply(&context).unwrap(), Value::Bool(false)); diff --git a/reductstore/src/storage/query/condition/operators/misc/cast.rs b/reductstore/src/storage/query/condition/operators/misc/cast.rs index 8f74ea2b7..8e9879511 100644 --- a/reductstore/src/storage/query/condition/operators/misc/cast.rs +++ b/reductstore/src/storage/query/condition/operators/misc/cast.rs @@ -14,7 +14,7 @@ pub(crate) struct Cast { impl Node for Cast { fn apply(&mut self, context: &Context) -> Result { let op = self.operands[0].apply(context)?; - let type_name = self.operands[1].apply(context)?.as_string()?; + let type_name = self.operands[1].apply(context)?.to_string(); op.cast(type_name.as_str()) } diff --git a/reductstore/src/storage/query/condition/operators/misc/exists.rs b/reductstore/src/storage/query/condition/operators/misc/exists.rs index 0eee8639a..24a7e4157 100644 --- a/reductstore/src/storage/query/condition/operators/misc/exists.rs +++ b/reductstore/src/storage/query/condition/operators/misc/exists.rs @@ -15,7 +15,7 @@ impl Node for Exists { fn apply(&mut self, context: &Context) -> Result { for operand in &mut self.operands { let value = operand.apply(context)?; - if !context.labels.contains_key(value.as_string()?.as_str()) { + if !context.labels.contains_key(value.to_string().as_str()) { return Ok(Value::Bool(false)); } } diff --git a/reductstore/src/storage/query/condition/operators/misc/ref.rs b/reductstore/src/storage/query/condition/operators/misc/ref.rs index 1ee710bbf..5b57de0c3 100644 --- a/reductstore/src/storage/query/condition/operators/misc/ref.rs +++ b/reductstore/src/storage/query/condition/operators/misc/ref.rs @@ -13,7 +13,7 @@ pub(crate) struct Ref { impl Node for Ref { fn apply(&mut self, context: &Context) -> Result { - let label = self.operands[0].apply(context)?.as_string()?; + let label = self.operands[0].apply(context)?.to_string(); context.labels.get(label.as_str()).map_or_else( || Err(not_found!("Label '{:?}' not found", label)), |v| Ok(Value::parse(v)), diff --git a/reductstore/src/storage/query/condition/parser.rs b/reductstore/src/storage/query/condition/parser.rs index d1c9326d3..0b360a574 100644 --- a/reductstore/src/storage/query/condition/parser.rs +++ b/reductstore/src/storage/query/condition/parser.rs @@ -12,7 +12,7 @@ use crate::storage::query::condition::operators::logical::{AllOf, AnyOf, In, Nin use crate::storage::query::condition::operators::misc::{Cast, Exists, Ref, Timestamp}; use crate::storage::query::condition::operators::string::{Contains, EndsWith, StartsWith}; use crate::storage::query::condition::reference::Reference; -use crate::storage::query::condition::value::Value; +use crate::storage::query::condition::value::{parse_duration, Value}; use crate::storage::query::condition::{Boxed, BoxedNode}; use reduct_base::error::ReductError; use reduct_base::unprocessable_entity; @@ -93,8 +93,9 @@ impl Parser { } else if value.starts_with("@") { Ok(vec![ComputedReference::boxed(value[1..].to_string())]) } else if value.starts_with("$") { - // operator without operands (nullary) Ok(vec![Self::parse_operator(value, vec![])?]) + } else if let Ok(duration) = parse_duration(value) { + Ok(vec![Constant::boxed(duration)]) } else { Ok(vec![Constant::boxed(Value::String(value.to_string()))]) } @@ -240,7 +241,16 @@ mod tests { #[rstest] fn test_parse_string(parser: Parser, context: Context) { let json = json!({ - "$and": [ "a","b"] + "$and": ["a","b"] + }); + let mut node = parser.parse(&json).unwrap(); + assert!(node.apply(&context).unwrap().as_bool().unwrap()); + } + + #[rstest] + fn test_parse_duration(parser: Parser, context: Context) { + let json = json!({ + "$eq": ["1h", 3600_000_000u64] }); let mut node = parser.parse(&json).unwrap(); assert!(node.apply(&context).unwrap().as_bool().unwrap()); @@ -405,7 +415,7 @@ mod tests { ) { let json = serde_json::from_str(&format!( r#"{{"$eq":[{}, {{"{}": {} }}] }}"#, - expected.as_string().unwrap(), + expected.to_string(), operator, operands )) diff --git a/reductstore/src/storage/query/condition/value.rs b/reductstore/src/storage/query/condition/value.rs index ef19e28aa..f8138f848 100644 --- a/reductstore/src/storage/query/condition/value.rs +++ b/reductstore/src/storage/query/condition/value.rs @@ -4,25 +4,29 @@ mod cmp; mod arithmetic; +mod duration_format; mod misc; mod string; use reduct_base::error::ReductError; use reduct_base::unprocessable_entity; +use std::fmt::Display; -pub(crate) use arithmetic::abs::Abs; -pub(crate) use arithmetic::add::Add; -pub(crate) use arithmetic::div::Div; -pub(crate) use arithmetic::div_num::DivNum; -pub(crate) use arithmetic::mult::Mult; -pub(crate) use arithmetic::rem::Rem; -pub(crate) use arithmetic::sub::Sub; +pub(super) use arithmetic::abs::Abs; +pub(super) use arithmetic::add::Add; +pub(super) use arithmetic::div::Div; +pub(super) use arithmetic::div_num::DivNum; +pub(super) use arithmetic::mult::Mult; +pub(super) use arithmetic::rem::Rem; +pub(super) use arithmetic::sub::Sub; -pub(crate) use string::contains::Contains; -pub(crate) use string::ends_with::EndsWith; -pub(crate) use string::starts_with::StartsWith; +pub(super) use string::contains::Contains; +pub(super) use string::ends_with::EndsWith; +pub(super) use string::starts_with::StartsWith; -pub(crate) use misc::cast::Cast; +use crate::storage::query::condition::value::duration_format::fmt_duration; +pub(super) use crate::storage::query::condition::value::duration_format::parse_duration; +pub(super) use misc::cast::Cast; /// A value that can be used in a condition. #[derive(Debug, Clone)] @@ -31,6 +35,7 @@ pub(crate) enum Value { Int(i64), Float(f64), String(String), + Duration(i64), } impl Value { @@ -61,7 +66,7 @@ impl Value { pub fn as_bool(&self) -> Result { match self { Value::Bool(value) => Ok(*value), - Value::Int(value) => Ok(value != &0), + Value::Int(value) | Value::Duration(value) => Ok(value != &0), Value::Float(value) => Ok(value != &0.0), Value::String(value) => Ok(!value.is_empty()), } @@ -72,7 +77,7 @@ impl Value { pub fn as_int(&self) -> Result { match self { Value::Bool(value) => Ok(*value as i64), - Value::Int(value) => Ok(*value), + Value::Int(value) | Value::Duration(value) => Ok(*value), Value::Float(value) => Ok(*value as i64), Value::String(value) => { if let Ok(value) = value.parse::() { @@ -91,7 +96,7 @@ impl Value { pub fn as_float(&self) -> Result { match self { Value::Bool(value) => Ok(*value as i64 as f64), - Value::Int(value) => Ok(*value as f64), + Value::Int(value) | Value::Duration(value) => Ok(*value as f64), Value::Float(value) => Ok(*value), Value::String(value) => { if let Ok(value) = value.parse::() { @@ -106,26 +111,35 @@ impl Value { } } - /// Converts the value to a string. - #[allow(dead_code)] - pub fn as_string(&self) -> Result { + /// Check if it is a string + pub fn is_string(&self) -> bool { match self { - Value::Bool(value) => Ok(value.to_string()), - Value::Int(value) => Ok(value.to_string()), - Value::Float(value) => Ok(value.to_string()), - Value::String(value) => Ok(value.clone()), + Value::String(_) => true, + _ => false, } } - /// Check if it is a string - pub fn is_string(&self) -> bool { + /// Check if it is a duration + pub fn is_duration(&self) -> bool { match self { - Value::String(_) => true, + Value::Duration(_) => true, _ => false, } } } +impl Display for Value { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Value::Bool(value) => write!(f, "{}", value), + Value::Int(value) => write!(f, "{}", value), + Value::Float(value) => write!(f, "{}", value), + Value::String(value) => write!(f, "{}", value), + Value::Duration(value) => fmt_duration(*value, f), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -218,13 +232,13 @@ mod tests { } #[rstest] - #[case(Value::Bool(true), Ok("true".to_string()))] - #[case(Value::Bool(false), Ok("false".to_string()))] - #[case(Value::Int(42), Ok("42".to_string()))] - #[case(Value::Float(42.0), Ok("42".to_string()))] - #[case(Value::String("string".to_string()), Ok("string".to_string()))] - fn as_string(#[case] value: Value, #[case] expected: Result) { - let result = value.as_string(); + #[case(Value::Bool(true), "true".to_string())] + #[case(Value::Bool(false), "false".to_string())] + #[case(Value::Int(42), "42".to_string())] + #[case(Value::Float(42.0), "42".to_string())] + #[case(Value::String("string".to_string()), "string".to_string())] + fn to_string(#[case] value: Value, #[case] expected: String) { + let result = value.to_string(); assert_eq!(result, expected); } } diff --git a/reductstore/src/storage/query/condition/value/arithmetic/abs.rs b/reductstore/src/storage/query/condition/value/arithmetic/abs.rs index 658bf8c39..5a5bce78e 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/abs.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/abs.rs @@ -18,7 +18,7 @@ impl Abs for Value { let mut result = self; match result { Value::Bool(s) => result = Value::Int(s as i64), - Value::Int(s) => result = Value::Int(s.abs()), + Value::Int(s) | Value::Duration(s) => result = Value::Int(s.abs()), Value::Float(s) => result = Value::Float(s.abs()), Value::String(_) => { return Err(unprocessable_entity!( @@ -43,6 +43,7 @@ mod tests { #[case(Value::Float(-1.0), Value::Float(1.0))] #[case(Value::Float(0.0), Value::Float(0.0))] #[case(Value::Float(1.0), Value::Float(1.0))] + #[case(Value::Duration(-1), Value::Duration(1))] fn test_abs(#[case] value: Value, #[case] expected: Value) { let result = value.abs().unwrap(); assert_eq!(result, expected); diff --git a/reductstore/src/storage/query/condition/value/arithmetic/add.rs b/reductstore/src/storage/query/condition/value/arithmetic/add.rs index a54f0395b..40975acbf 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/add.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/add.rs @@ -28,25 +28,28 @@ impl Add for Value { match result { Value::Bool(s) => match other { Value::Bool(v) => result = Value::Int(s as i64 + v as i64), - Value::Int(v) => result = Value::Int(s as i64 + v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s as i64 + v), Value::Float(v) => result = Value::Float(s as i8 as f64 + v), Value::String(_) => { return Err(unprocessable_entity!("Cannot add boolean to string")); } }, - Value::Int(s) => match other { + Value::Int(s) | Value::Duration(s) => match other { Value::Bool(v) => result = Value::Int(s + v as i64), - Value::Int(v) => result = Value::Int(s + v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s + v), Value::Float(v) => result = Value::Float(s as f64 + v), Value::String(_) => { + if let Value::Duration(_) = result { + return Err(unprocessable_entity!("Cannot add duration to string")); + } return Err(unprocessable_entity!("Cannot add integer to string")); } }, Value::Float(s) => match other { Value::Bool(v) => result = Value::Float(s + v as i8 as f64), - Value::Int(v) => result = Value::Float(s + v as f64), + Value::Int(v) | Value::Duration(v) => result = Value::Float(s + v as f64), Value::Float(v) => result = Value::Float(s + v), Value::String(_) => { return Err(unprocessable_entity!("Cannot add float to string")); @@ -65,6 +68,9 @@ impl Add for Value { return Err(unprocessable_entity!("Cannot add string to float")); } Value::String(v) => result = Value::String(format!("{}{}", s, v)), + Value::Duration(_) => { + return Err(unprocessable_entity!("Cannot add string to duration")); + } }, } @@ -80,6 +86,7 @@ mod tests { #[case(Value::Bool(true), Value::Bool(false), Value::Int(1))] #[case(Value::Bool(true), Value::Int(2), Value::Int(3))] #[case(Value::Bool(true), Value::Float(2.0), Value::Float(3.0))] + #[case(Value::Bool(true), Value::Duration(2), Value::Duration(3))] #[case(Value::Int(2), Value::Bool(true), Value::Int(3))] #[case(Value::Int(2), Value::Int(3), Value::Int(5))] #[case(Value::Int(2), Value::Float(3.0), Value::Float(5.0))] @@ -95,9 +102,11 @@ mod tests { #[case(Value::Bool(true), Value::String("world".to_string()), unprocessable_entity!("Cannot add boolean to string"))] #[case(Value::Int(2), Value::String("world".to_string()), unprocessable_entity!("Cannot add integer to string"))] #[case(Value::Float(2.0), Value::String("world".to_string()), unprocessable_entity!("Cannot add float to string"))] + #[case(Value::Duration(2), Value::String("world".to_string()), unprocessable_entity!("Cannot add duration to string"))] #[case(Value::String("hello".to_string()), Value::Bool(true), unprocessable_entity!("Cannot add string to boolean"))] #[case(Value::String("hello".to_string()), Value::Int(2), unprocessable_entity!("Cannot add string to integer"))] #[case(Value::String("hello".to_string()), Value::Float(2.0), unprocessable_entity!("Cannot add string to float"))] + #[case(Value::String("hello".to_string()), Value::Duration(2), unprocessable_entity!("Cannot add string to duration"))] fn test_add_err(#[case] a: Value, #[case] b: Value, #[case] expected: ReductError) { assert_eq!(a.add(b), Err(expected)); } diff --git a/reductstore/src/storage/query/condition/value/arithmetic/div.rs b/reductstore/src/storage/query/condition/value/arithmetic/div.rs index 388a4e0f6..a5e92c8da 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/div.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/div.rs @@ -52,12 +52,15 @@ mod tests { #[case(Value::Bool(true), Value::Bool(true), Value::Float(1.0))] #[case(Value::Bool(true), Value::Int(2), Value::Float(0.5))] #[case(Value::Bool(true), Value::Float(3.0), Value::Float(1.0 / 3.0))] + #[case(Value::Bool(true), Value::Duration(4), Value::Float(0.25))] #[case(Value::Int(4), Value::Bool(true), Value::Float(4.0))] #[case(Value::Int(5), Value::Int(2), Value::Float(2.5))] #[case(Value::Int(6), Value::Float(3.0), Value::Float(2.0))] + #[case(Value::Int(7), Value::Duration(2), Value::Float(3.5))] #[case(Value::Float(7.0), Value::Bool(true), Value::Float(7.0))] #[case(Value::Float(8.0), Value::Int(2), Value::Float(4.0))] #[case(Value::Float(9.0), Value::Float(3.0), Value::Float(3.0))] + #[case(Value::Float(10.0), Value::Duration(2), Value::Float(5.0))] fn divide_ok(#[case] value: Value, #[case] other: Value, #[case] expected: Value) { let result = value.divide(other).unwrap(); assert_eq!(result, expected); @@ -67,6 +70,7 @@ mod tests { #[case(Value::Bool(true), Value::String("xxx".to_string()))] #[case(Value::Int(1), Value::String("xxx".to_string()))] #[case(Value::Float(2.0), Value::String("xxx".to_string()))] + #[case( Value::Duration(3), Value::String("xxx".to_string()))] fn divide_by_string(#[case] value: Value, #[case] other: Value) { let result = value.divide(other); assert_eq!( @@ -80,6 +84,7 @@ mod tests { #[case(Value::String("xxx".to_string()), Value::Int(1))] #[case(Value::String("xxx".to_string()), Value::Float(2.0))] #[case(Value::String("xxx".to_string()), Value::String("xxx".to_string()))] + #[case(Value::String("xxx".to_string()), Value::Duration(3))] fn divide_string(#[case] value: Value, #[case] other: Value) { let result = value.divide(other); assert_eq!(result, Err(unprocessable_entity!("Cannot divide string"))); diff --git a/reductstore/src/storage/query/condition/value/arithmetic/div_num.rs b/reductstore/src/storage/query/condition/value/arithmetic/div_num.rs index f476b6bcc..6dd7faa41 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/div_num.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/div_num.rs @@ -52,12 +52,15 @@ mod tests { #[case(Value::Bool(true), Value::Bool(true), Value::Int(1))] #[case(Value::Bool(true), Value::Int(2), Value::Int(0))] #[case(Value::Bool(true), Value::Float(3.0), Value::Int(0))] + #[case(Value::Bool(true), Value::Duration(4), Value::Int(0))] #[case(Value::Int(4), Value::Bool(true), Value::Int(4))] #[case(Value::Int(5), Value::Int(2), Value::Int(2))] #[case(Value::Int(6), Value::Float(3.0), Value::Int(2))] + #[case(Value::Int(7), Value::Duration(2), Value::Int(3))] #[case(Value::Float(7.0), Value::Bool(true), Value::Int(7))] #[case(Value::Float(8.0), Value::Int(2), Value::Int(4))] #[case(Value::Float(9.0), Value::Float(3.0), Value::Int(3))] + #[case(Value::Float(10.0), Value::Duration(2), Value::Int(5))] fn divide_ok(#[case] value: Value, #[case] other: Value, #[case] expected: Value) { let result = value.divide_num(other).unwrap(); assert_eq!(result, expected); @@ -67,6 +70,7 @@ mod tests { #[case(Value::Bool(true), Value::String("xxx".to_string()))] #[case(Value::Int(1), Value::String("xxx".to_string()))] #[case(Value::Float(2.0), Value::String("xxx".to_string()))] + #[case( Value::Duration(3), Value::String("xxx".to_string()))] fn divide_by_string(#[case] value: Value, #[case] other: Value) { let result = value.divide_num(other); assert_eq!( @@ -80,6 +84,7 @@ mod tests { #[case(Value::String("xxx".to_string()), Value::Int(1))] #[case(Value::String("xxx".to_string()), Value::Float(2.0))] #[case(Value::String("xxx".to_string()), Value::String("xxx".to_string()))] + #[case(Value::String("xxx".to_string()), Value::Duration(3))] fn divide_string(#[case] value: Value, #[case] other: Value) { let result = value.divide_num(other); assert_eq!(result, Err(unprocessable_entity!("Cannot divide string"))); @@ -88,6 +93,7 @@ mod tests { #[rstest] #[case(Value::Int(1), Value::Int(0))] #[case(Value::Float(1.0), Value::Float(0.0))] + #[case(Value::Duration(1), Value::Duration(0))] fn divide_by_zero(#[case] value: Value, #[case] other: Value) { let result = value.divide_num(other); assert_eq!(result, Err(unprocessable_entity!("Cannot divide by zero"))); diff --git a/reductstore/src/storage/query/condition/value/arithmetic/mult.rs b/reductstore/src/storage/query/condition/value/arithmetic/mult.rs index d8417058a..1f9c07553 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/mult.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/mult.rs @@ -28,25 +28,28 @@ impl Mult for Value { match result { Value::Bool(s) => match other { Value::Bool(v) => result = Value::Int(s as i64 * v as i64), - Value::Int(v) => result = Value::Int(s as i64 * v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s as i64 * v), Value::Float(v) => result = Value::Float(s as i8 as f64 * v), Value::String(_) => { return Err(unprocessable_entity!("Cannot multiply boolean by string")); } }, - Value::Int(s) => match other { + Value::Int(s) | Value::Duration(s) => match other { Value::Bool(v) => result = Value::Int(s * v as i64), - Value::Int(v) => result = Value::Int(s * v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s * v), Value::Float(v) => result = Value::Float(s as f64 * v), Value::String(_) => { + if let Value::Duration(_) = result { + return Err(unprocessable_entity!("Cannot multiply duration by string")); + } return Err(unprocessable_entity!("Cannot multiply integer by string")); } }, Value::Float(s) => match other { Value::Bool(v) => result = Value::Float(s * v as i8 as f64), - Value::Int(v) => result = Value::Float(s * v as f64), + Value::Int(v) | Value::Duration(v) => result = Value::Float(s * v as f64), Value::Float(v) => result = Value::Float(s * v), Value::String(_) => { return Err(unprocessable_entity!("Cannot multiply float by string")); @@ -68,6 +71,9 @@ impl Mult for Value { Value::String(_) => { Err(unprocessable_entity!("Cannot multiply string by string")) } + Value::Duration(_) => { + Err(unprocessable_entity!("Cannot multiply string by duration")) + } } } } @@ -84,12 +90,15 @@ mod tests { #[case(Value::Bool(true), Value::Bool(false), Value::Int(0))] #[case(Value::Bool(true), Value::Int(2), Value::Int(2))] #[case(Value::Bool(true), Value::Float(2.0), Value::Float(2.0))] + #[case(Value::Bool(true), Value::Duration(2), Value::Duration(2))] #[case(Value::Int(2), Value::Bool(true), Value::Int(2))] #[case(Value::Int(2), Value::Int(2), Value::Int(4))] #[case(Value::Int(2), Value::Float(2.0), Value::Float(4.0))] + #[case(Value::Int(2), Value::Duration(2), Value::Duration(4))] #[case(Value::Float(2.0), Value::Bool(true), Value::Float(2.0))] #[case(Value::Float(2.0), Value::Int(2), Value::Float(4.0))] #[case(Value::Float(2.0), Value::Float(2.0), Value::Float(4.0))] + #[case(Value::Float(2.0), Value::Duration(2), Value::Duration(4))] fn multiply(#[case] value: Value, #[case] other: Value, #[case] expected: Value) { let result = value.multiply(other).unwrap(); assert_eq!(result, expected); @@ -99,10 +108,12 @@ mod tests { #[case(Value::Bool(true), Value::String("string".to_string()), unprocessable_entity!("Cannot multiply boolean by string"))] #[case(Value::Int(2), Value::String("string".to_string()), unprocessable_entity!("Cannot multiply integer by string"))] #[case(Value::Float(2.0), Value::String("string".to_string()), unprocessable_entity!("Cannot multiply float by string"))] + #[case(Value::Duration(2), Value::String("string".to_string()), unprocessable_entity!("Cannot multiply duration by string"))] #[case(Value::String("string".to_string()), Value::Bool(true), unprocessable_entity!("Cannot multiply string by boolean"))] #[case(Value::String("string".to_string()), Value::Int(2), unprocessable_entity!("Cannot multiply string by integer"))] #[case(Value::String("string".to_string()), Value::Float(2.0), unprocessable_entity!("Cannot multiply string by float"))] #[case(Value::String("string".to_string()), Value::String("string".to_string()), unprocessable_entity!("Cannot multiply string by string"))] + #[case(Value::String("string".to_string()), Value::Duration(2), unprocessable_entity!("Cannot multiply string by duration"))] fn multiply_error(#[case] value: Value, #[case] other: Value, #[case] expected: ReductError) { let result = value.multiply(other); assert_eq!(result, Err(expected)); diff --git a/reductstore/src/storage/query/condition/value/arithmetic/rem.rs b/reductstore/src/storage/query/condition/value/arithmetic/rem.rs index 76066c227..65f5d19aa 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/rem.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/rem.rs @@ -28,25 +28,28 @@ impl Rem for Value { match result { Value::Bool(s) => match other { Value::Bool(v) => result = Value::Int(s as i64 % v as i64), - Value::Int(v) => result = Value::Int(s as i64 % v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s as i64 % v), Value::Float(v) => result = Value::Float(s as i8 as f64 % v), Value::String(_) => { return Err(unprocessable_entity!("Cannot divide boolean by string")); } }, - Value::Int(s) => match other { + Value::Int(s) | Value::Duration(s) => match other { Value::Bool(v) => result = Value::Int(s % v as i64), - Value::Int(v) => result = Value::Int(s % v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s % v), Value::Float(v) => result = Value::Float(s as f64 % v), Value::String(_) => { + if let Value::Duration(_) = result { + return Err(unprocessable_entity!("Cannot divide duration by string")); + } return Err(unprocessable_entity!("Cannot divide integer by string")); } }, Value::Float(s) => match other { Value::Bool(v) => result = Value::Float(s % v as i8 as f64), - Value::Int(v) => result = Value::Float(s % v as f64), + Value::Int(v) | Value::Duration(v) => result = Value::Float(s % v as f64), Value::Float(v) => result = Value::Float(s % v), Value::String(_) => { return Err(unprocessable_entity!("Cannot divide float by string")); @@ -57,9 +60,11 @@ impl Rem for Value { return match other { Value::Bool(_) => Err(unprocessable_entity!("Cannot divide string by boolean")), Value::Int(_) => Err(unprocessable_entity!("Cannot divide string by integer")), - Value::Float(_) => Err(unprocessable_entity!("Cannot divide string by float")), Value::String(_) => Err(unprocessable_entity!("Cannot divide string")), + Value::Duration(_) => { + Err(unprocessable_entity!("Cannot divide string by duration")) + } } } } @@ -76,9 +81,11 @@ mod tests { #[case(Value::Bool(true), Value::Bool(true), Value::Int(0))] #[case(Value::Bool(true), Value::Int(2), Value::Int(1))] #[case(Value::Bool(true), Value::Float(3.0), Value::Float(1.0))] + #[case(Value::Bool(true), Value::Duration(4), Value::Float(1.0))] #[case(Value::Int(4), Value::Bool(true), Value::Int(0))] #[case(Value::Int(5), Value::Int(2), Value::Int(1))] #[case(Value::Int(6), Value::Float(3.5), Value::Float(2.5))] + #[case(Value::Int(7), Value::Duration(2), Value::Float(1.0))] #[case(Value::Float(7.0), Value::Bool(true), Value::Float(0.0))] #[case(Value::Float(8.0), Value::Int(3), Value::Float(2.0))] #[case(Value::Float(-9.0), Value::Float(3.5), Value::Float(-2.0))] @@ -91,10 +98,12 @@ mod tests { #[case(Value::Bool(true), Value::String("string".to_string()), unprocessable_entity!("Cannot divide boolean by string"))] #[case(Value::Int(1), Value::String("string".to_string()), unprocessable_entity!("Cannot divide integer by string"))] #[case(Value::Float(2.0), Value::String("string".to_string()), unprocessable_entity!("Cannot divide float by string"))] + #[case( Value::Duration(3), Value::String("string".to_string()), unprocessable_entity!("Cannot divide duration by string"))] #[case(Value::String("string".to_string()), Value::Bool(true), unprocessable_entity!("Cannot divide string by boolean"))] #[case(Value::String("string".to_string()), Value::Int(1), unprocessable_entity!("Cannot divide string by integer"))] #[case(Value::String("string".to_string()), Value::Float(2.0), unprocessable_entity!("Cannot divide string by float"))] #[case(Value::String("string".to_string()), Value::String("string".to_string()), unprocessable_entity!("Cannot divide string"))] + #[case(Value::String("string".to_string()), Value::Duration(3), unprocessable_entity!("Cannot divide string by duration"))] fn rem_err(#[case] value: Value, #[case] other: Value, #[case] expected: ReductError) { let result = value.remainder(other); assert_eq!(result, Err(expected)); diff --git a/reductstore/src/storage/query/condition/value/arithmetic/sub.rs b/reductstore/src/storage/query/condition/value/arithmetic/sub.rs index cada61fb2..0bb4266a4 100644 --- a/reductstore/src/storage/query/condition/value/arithmetic/sub.rs +++ b/reductstore/src/storage/query/condition/value/arithmetic/sub.rs @@ -28,25 +28,30 @@ impl Sub for Value { match result { Value::Bool(s) => match other { Value::Bool(v) => result = Value::Int(s as i64 - v as i64), - Value::Int(v) => result = Value::Int(s as i64 - v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s as i64 - v), Value::Float(v) => result = Value::Float(s as i8 as f64 - v), Value::String(_) => { return Err(unprocessable_entity!("Cannot subtract string from boolean")); } }, - Value::Int(s) => match other { + Value::Int(s) | Value::Duration(s) => match other { Value::Bool(v) => result = Value::Int(s - v as i64), - Value::Int(v) => result = Value::Int(s - v), + Value::Int(v) | Value::Duration(v) => result = Value::Int(s - v), Value::Float(v) => result = Value::Float(s as f64 - v), Value::String(_) => { + if let Value::Duration(_) = result { + return Err(unprocessable_entity!( + "Cannot subtract string from duration" + )); + } return Err(unprocessable_entity!("Cannot subtract string from integer")); } }, Value::Float(s) => match other { Value::Bool(v) => result = Value::Float(s - v as i8 as f64), - Value::Int(v) => result = Value::Float(s - v as f64), + Value::Int(v) | Value::Duration(v) => result = Value::Float(s - v as f64), Value::Float(v) => result = Value::Float(s - v), Value::String(_) => { return Err(unprocessable_entity!("Cannot subtract string from float")); @@ -66,6 +71,9 @@ impl Sub for Value { Err(unprocessable_entity!("Cannot subtract string from float")) } Value::String(_) => Err(unprocessable_entity!("Cannot subtract string")), + Value::Duration(_) => Err(unprocessable_entity!( + "Cannot subtract string from duration" + )), } } } @@ -82,12 +90,15 @@ mod tests { #[case(Value::Bool(true), Value::Bool(false), Value::Int(1))] #[case(Value::Bool(true), Value::Int(1), Value::Int(0))] #[case(Value::Bool(true), Value::Float(1.0), Value::Float(0.0))] + #[case(Value::Bool(true), Value::Duration(1), Value::Float(0.0))] #[case(Value::Int(1), Value::Bool(true), Value::Int(0))] #[case(Value::Int(1), Value::Int(1), Value::Int(0))] #[case(Value::Int(1), Value::Float(1.0), Value::Float(0.0))] + #[case(Value::Int(1), Value::Duration(1), Value::Float(0.0))] #[case(Value::Float(1.0), Value::Bool(true), Value::Float(0.0))] #[case(Value::Float(1.0), Value::Int(1), Value::Float(0.0))] #[case(Value::Float(1.0), Value::Float(1.0), Value::Float(0.0))] + #[case(Value::Float(1.0), Value::Duration(1), Value::Float(0.0))] fn sub(#[case] value: Value, #[case] other: Value, #[case] expected: Value) { let result = value.subtract(other).unwrap(); assert_eq!(result, expected); @@ -97,10 +108,12 @@ mod tests { #[case(Value::Bool(true), Value::String("string".to_string()), unprocessable_entity!("Cannot subtract string from boolean"))] #[case(Value::Int(1), Value::String("string".to_string()), unprocessable_entity!("Cannot subtract string from integer"))] #[case(Value::Float(1.0), Value::String("string".to_string()), unprocessable_entity!("Cannot subtract string from float"))] + #[case(Value::Duration(1), Value::String("string".to_string()), unprocessable_entity!("Cannot subtract string from duration"))] #[case(Value::String("string".to_string()), Value::Bool(true), unprocessable_entity!("Cannot subtract string from boolean"))] #[case(Value::String("string".to_string()), Value::Int(1), unprocessable_entity!("Cannot subtract string from integer"))] #[case(Value::String("string".to_string()), Value::Float(1.0), unprocessable_entity!("Cannot subtract string from float"))] #[case(Value::String("string".to_string()), Value::String("string".to_string()), unprocessable_entity!("Cannot subtract string"))] + #[case(Value::String("string".to_string()), Value::Duration(1), unprocessable_entity!("Cannot subtract string from duration"))] fn sub_err(#[case] value: Value, #[case] other: Value, #[case] expected: ReductError) { let result = value.subtract(other); diff --git a/reductstore/src/storage/query/condition/value/cmp.rs b/reductstore/src/storage/query/condition/value/cmp.rs index d0aa0ff64..bf4bda5a6 100644 --- a/reductstore/src/storage/query/condition/value/cmp.rs +++ b/reductstore/src/storage/query/condition/value/cmp.rs @@ -7,15 +7,26 @@ impl PartialEq for Value { fn eq(&self, other: &Self) -> bool { match (self, other) { (Value::Bool(left), Value::Bool(right)) => left == right, - (Value::Bool(left), Value::Int(right)) => *left as i64 == *right, + (Value::Bool(left), Value::Int(right) | Value::Duration(right)) => { + *left as i64 == *right + } (Value::Bool(left), Value::Float(right)) => *left as i64 as f64 == *right, - (Value::Int(left), Value::Int(right)) => left == right, - (Value::Int(left), Value::Bool(right)) => *left == *right as i64, - (Value::Int(left), Value::Float(right)) => *left as f64 == *right, + ( + Value::Int(left) | Value::Duration(left), + Value::Int(right) | Value::Duration(right), + ) => left == right, + (Value::Int(left) | Value::Duration(left), Value::Bool(right)) => { + *left == *right as i64 + } + (Value::Int(left) | Value::Duration(left), Value::Float(right)) => { + *left as f64 == *right + } (Value::Float(left), Value::Float(right)) => left == right, - (Value::Float(left), Value::Int(right)) => *left == *right as f64, + (Value::Float(left), Value::Int(right) | Value::Duration(right)) => { + *left == *right as f64 + } (Value::Float(left), Value::Bool(right)) => *left == *right as i64 as f64, (Value::String(left), Value::String(right)) => left == right, @@ -29,15 +40,26 @@ impl PartialOrd for Value { fn partial_cmp(&self, other: &Self) -> Option { match (self, other) { (Value::Bool(left), Value::Bool(right)) => left.partial_cmp(right), - (Value::Bool(left), Value::Int(right)) => (*left as i64).partial_cmp(right), + (Value::Bool(left), Value::Int(right) | Value::Duration(right)) => { + (*left as i64).partial_cmp(right) + } (Value::Bool(left), Value::Float(right)) => (*left as i64 as f64).partial_cmp(right), - (Value::Int(left), Value::Int(right)) => left.partial_cmp(right), - (Value::Int(left), Value::Bool(right)) => left.partial_cmp(&(*right as i64)), - (Value::Int(left), Value::Float(right)) => (*left as f64).partial_cmp(right), + ( + Value::Int(left) | Value::Duration(left), + Value::Int(right) | Value::Duration(right), + ) => left.partial_cmp(right), + (Value::Int(left) | Value::Duration(left), Value::Bool(right)) => { + left.partial_cmp(&(*right as i64)) + } + (Value::Int(left) | Value::Duration(left), Value::Float(right)) => { + (*left as f64).partial_cmp(right) + } (Value::Float(left), Value::Float(right)) => left.partial_cmp(right), - (Value::Float(left), Value::Int(right)) => left.partial_cmp(&(*right as f64)), + (Value::Float(left), Value::Int(right) | Value::Duration(right)) => { + left.partial_cmp(&(*right as f64)) + } (Value::Float(left), Value::Bool(right)) => left.partial_cmp(&(*right as i64 as f64)), (Value::String(left), Value::String(right)) => left.partial_cmp(right), @@ -67,6 +89,7 @@ mod tests { #[case(Value::Bool(true), Value::String("string".to_string()), false)] #[case(Value::Bool(false), Value::String("string".to_string()), false)] #[case(Value::Bool(true), Value::String("true".to_string()), false)] + #[case(Value::Bool(true), Value::Duration(1), true)] fn partial_eq_bool(#[case] left: Value, #[case] right: Value, #[case] expected: bool) { let result = left == right; assert_eq!(result, expected); @@ -88,6 +111,8 @@ mod tests { #[case(Value::Int(1), Value::Float(-1.0), false)] #[case(Value::Int(1), Value::String("string".to_string()), false)] #[case(Value::Int(-1), Value::String("string".to_string()), false)] + #[case(Value::Int(1), Value::Duration(1), true)] + #[case(Value::Int(-1), Value::Duration(1), false)] fn partial_eq_int(#[case] left: Value, #[case] right: Value, #[case] expected: bool) { let result = left == right; assert_eq!(result, expected); @@ -109,6 +134,8 @@ mod tests { #[case(Value::Float(1.0), Value::Int(-1), false)] #[case(Value::Float(1.0), Value::String("string".to_string()), false)] #[case(Value::Float(-1.0), Value::String("string".to_string()), false)] + #[case(Value::Float(1.0), Value::Duration(1), true)] + #[case(Value::Float(-1.0), Value::Duration(1), false)] fn partial_eq_float(#[case] left: Value, #[case] right: Value, #[case] expected: bool) { let result = left == right; assert_eq!(result, expected); @@ -122,6 +149,7 @@ mod tests { #[case(Value::String("a".to_string()), Value::Bool(false), false)] #[case(Value::String("a".to_string()), Value::Int(1), false)] #[case(Value::String("a".to_string()), Value::Float(1.0), false)] + #[case(Value::String("a".to_string()), Value::Duration(1), false)] fn partial_eq_string(#[case] left: Value, #[case] right: Value, #[case] expected: bool) { let result = left == right; assert_eq!(result, expected); @@ -154,6 +182,9 @@ mod tests { #[case(Value::Bool(false), Value::String("string".to_string()), None)] #[case(Value::Bool(true), Value::String("true".to_string()), None)] #[case(Value::Bool(false), Value::String("true".to_string()), None)] + #[case(Value::Bool(true), Value::Duration(1), Some(Ordering::Equal))] + #[case(Value::Bool(false), Value::Duration(1), Some(Ordering::Less))] + #[case(Value::Bool(true), Value::Duration(0), Some(Ordering::Greater))] fn partial_cmp_bool( #[case] left: Value, #[case] right: Value, @@ -180,6 +211,9 @@ mod tests { #[case(Value::Int(1), Value::Float(-1.0), Some(Ordering::Greater))] #[case(Value::Int(1), Value::String("string".to_string()), None)] #[case(Value::Int(-1), Value::String("string".to_string()), None)] + #[case(Value::Int(1), Value::Duration(1), Some(Ordering::Equal))] + #[case(Value::Int(-1), Value::Duration(1), Some(Ordering::Less))] + #[case(Value::Int(1), Value::Duration(0), Some(Ordering::Greater))] fn partial_cmp_int( #[case] left: Value, #[case] right: Value, @@ -206,6 +240,8 @@ mod tests { #[case(Value::Float(1.0), Value::Int(-1), Some(Ordering::Greater))] #[case(Value::Float(1.0), Value::String("string".to_string()), None)] #[case(Value::Float(-1.0), Value::String("string".to_string()), None)] + #[case(Value::Float(1.0), Value::Duration(1), Some(Ordering::Equal))] + #[case(Value::Float(-1.0), Value::Duration(1), Some(Ordering::Less))] fn partial_cmp_float( #[case] left: Value, #[case] right: Value, @@ -223,6 +259,7 @@ mod tests { #[case(Value::String("a".to_string()), Value::Bool(false), None)] #[case(Value::String("a".to_string()), Value::Int(1), None)] #[case(Value::String("a".to_string()), Value::Float(1.0), None)] + #[case(Value::String("a".to_string()), Value::Duration(1), None)] fn partial_cmp_string( #[case] left: Value, #[case] right: Value, diff --git a/reductstore/src/storage/query/condition/value/duration_format.rs b/reductstore/src/storage/query/condition/value/duration_format.rs new file mode 100644 index 000000000..05ae901a1 --- /dev/null +++ b/reductstore/src/storage/query/condition/value/duration_format.rs @@ -0,0 +1,177 @@ +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 +use crate::storage::query::condition::value::Value; +use reduct_base::error::ReductError; +use reduct_base::unprocessable_entity; + +/// Parses a duration string into a `Value::Duration`. +/// +/// # Arguments +/// +/// * `duration_string` - The string representing the duration, e.g., "100ms", "2.5m", "1h". +/// +/// # Returns +/// +/// A `Result` containing the parsed duration as i64 (in microseconds) or an error if the string is invalid. +fn parse_single_duration(duration_string: &str) -> Result { + if duration_string.trim().is_empty() { + return Err(unprocessable_entity!("Duration literal cannot be empty")); + } + + let duration_string = duration_string.trim(); + let (num_part, unit_part) = duration_string + .chars() + .partition::(|c| c.is_digit(10) || *c == '.' || *c == '-'); + let value: i64 = num_part + .parse() + .map_err(|_| unprocessable_entity!("Invalid duration value: {}", duration_string))?; + + let unit = unit_part.as_str(); + let seconds = match unit { + "us" => value, + "ms" => value * 1000, + "s" => value * 1_000_000, + "m" => value * 60_000_000, + "h" => value * 3_600_000_000, + "d" => value * 86_400_000_000, + _ => { + return Err(unprocessable_entity!( + "Invalid duration unit: {}", + unit_part + )) + } + }; + Ok(seconds) +} + +/// Parses a duration string containing multiple parts (e.g., "100ms 500us") into a `Value::Duration`. +/// # Arguments +/// +/// * `duration_string` - The string representing the duration, which can contain multiple parts separated by whitespace. +/// +/// # Returns +/// +/// A `Result` containing the total duration as `Value::Duration` or an error if the string is invalid. +pub(crate) fn parse_duration(duration_string: &str) -> Result { + if duration_string.trim().is_empty() { + return Err(unprocessable_entity!("Duration literal cannot be empty")); + } + + let mut total_seconds = 0; + for part in duration_string.split_whitespace() { + let seconds = parse_single_duration(part)?; + total_seconds += seconds; + } + Ok(Value::Duration(total_seconds)) +} + +/// Formats a duration in microseconds into a human-readable string. +/// +/// # Arguments +/// +/// * `usec` - The duration in microseconds. +/// * `f` - The formatter to write the output to. +/// +/// # Returns +/// +/// A `Result` indicating success or failure of the formatting operation. +pub(super) fn fmt_duration(mut usec: i64, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut parts = Vec::new(); + let units = [ + ("d", 86_400_000_000), // days + ("h", 3_600_000_000), // hours + ("m", 60_000_000), // minutes + ("s", 1_000_000), // seconds + ("ms", 1000), // milliseconds + ("us", 1), // microseconds + ]; + for &(unit, unit_seconds) in &units { + if usec.abs() >= unit_seconds { + let value = usec / unit_seconds; + parts.push(format!("{}{}", value, unit)); + usec -= value * unit_seconds; + } + } + if parts.is_empty() { + parts.push("0us".to_string()); + } + write!(f, "{}", parts.join(" ")) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case(0, "0us")] + #[case(1, "1us")] + #[case(-1, "-1us")] + #[case(100, "100us")] + #[case(100_000, "100ms")] + #[case(1_000_000, "1s")] + #[case(-1_000_000, "-1s")] + #[case(60_000_000, "1m")] + #[case(3_600_000_000, "1h")] + #[case(86_400_000_000, "1d")] + #[case(86_400_000_000 + 3_600_000_000, "1d 1h")] + #[case(86_400_000_000 - 3_600_000_000 + 5, "23h 5us")] + fn test_fmt_duration(#[case] value: i64, #[case] literal: &str) { + let value = Value::Duration(value); + assert_eq!(parse_duration(literal).unwrap(), value); + assert_eq!(value.to_string(), literal); + } + + #[rstest] + fn test_parse_invalid_duration() { + assert_eq!( + parse_single_duration("").err().unwrap(), + unprocessable_entity!("Duration literal cannot be empty") + ); + assert_eq!( + parse_single_duration("100xyz").err().unwrap(), + unprocessable_entity!("Invalid duration unit: xyz") + ); + assert_eq!( + parse_single_duration("abc").err().unwrap(), + unprocessable_entity!("Invalid duration value: abc") + ); + + assert_eq!( + parse_single_duration("2.5m").err().unwrap(), + unprocessable_entity!("Invalid duration value: 2.5m") + ); + } + + #[rstest] + fn test_parse_duration() { + assert_eq!( + parse_duration("100ms 500us").unwrap(), + Value::Duration(100_500) + ); + assert_eq!( + parse_duration("1h -30m").unwrap(), + Value::Duration(1_800_000_000) + ); + assert_eq!( + parse_duration("2d 3h").unwrap(), + Value::Duration(183_600_000_000) + ); + } + + #[rstest] + fn test_invalid_duration() { + assert_eq!( + parse_duration("").err().unwrap(), + unprocessable_entity!("Duration literal cannot be empty") + ); + assert_eq!( + parse_duration("1h 100xyz").err().unwrap(), + unprocessable_entity!("Invalid duration unit: xyz") + ); + assert_eq!( + parse_duration("1h,2m").err().unwrap(), + unprocessable_entity!("Invalid duration unit: h,m") + ); + } +} diff --git a/reductstore/src/storage/query/condition/value/misc/cast.rs b/reductstore/src/storage/query/condition/value/misc/cast.rs index 2787b7a4f..92c426bc1 100644 --- a/reductstore/src/storage/query/condition/value/misc/cast.rs +++ b/reductstore/src/storage/query/condition/value/misc/cast.rs @@ -27,7 +27,8 @@ impl Cast for Value { "bool" => self.as_bool().map(Value::Bool), "int" => self.as_int().map(Value::Int), "float" => self.as_float().map(Value::Float), - "string" => self.as_string().map(Value::String), + "string" => Ok(Value::String(self.to_string())), + "duration" => self.as_int().map(Value::Duration), _ => Err(unprocessable_entity!("Unknown type '{}'", type_name)), } } @@ -44,20 +45,29 @@ mod tests { #[case("bool", Value::Int(1), Ok(Value::Bool(true)))] #[case("bool", Value::Float(1.0), Ok(Value::Bool(true)))] #[case("bool", Value::String("true".to_string()), Ok(Value::Bool(true)))] + #[case("bool", Value::Duration(1), Ok(Value::Bool(true)))] #[case("int", Value::Bool(true), Ok(Value::Int(1)))] #[case("int", Value::Int(1), Ok(Value::Int(1)))] #[case("int", Value::Float(1.0), Ok(Value::Int(1)))] #[case("int", Value::String("1".to_string()), Ok(Value::Int(1)))] #[case("int", Value::String("xx".to_string()), Err(unprocessable_entity!("Value 'xx' could not be parsed as integer")))] + #[case("int", Value::Duration(1), Ok(Value::Int(1)))] #[case("float", Value::Bool(true), Ok(Value::Float(1.0)))] #[case("float", Value::Int(1), Ok(Value::Float(1.0)))] #[case("float", Value::Float(1.0), Ok(Value::Float(1.0)))] #[case("float", Value::String("1.0".to_string()), Ok(Value::Float(1.0)))] #[case("float", Value::String("xx".to_string()), Err(unprocessable_entity!("Value 'xx' could not be parsed as float")))] + #[case("float", Value::Duration(1), Ok(Value::Float(1.0)))] #[case("string", Value::Bool(true), Ok(Value::String("true".to_string())))] #[case("string", Value::Int(1), Ok(Value::String("1".to_string())))] #[case("string", Value::Float(1.0), Ok(Value::String("1".to_string())))] #[case("string", Value::String("1".to_string()), Ok(Value::String("1".to_string())))] + #[case("string", Value::Duration(1), Ok(Value::String("1us".to_string())))] + #[case("duration", Value::Bool(true), Ok(Value::Duration(1)))] + #[case("duration", Value::Int(1), Ok(Value::Duration(1)))] + #[case("duration", Value::Float(1.0), Ok(Value::Duration(1)))] + #[case("duration", Value::String("1".to_string()), Ok(Value::Duration(1)))] + #[case("duration", Value::String("xx".to_string()), Err(unprocessable_entity!("Value 'xx' could not be parsed as integer")))] #[case("unknown", Value::Bool(true), Err(unprocessable_entity!("Unknown type 'unknown'")))] fn test_cast( #[case] type_name: &str, diff --git a/reductstore/src/storage/query/condition/value/string/contains.rs b/reductstore/src/storage/query/condition/value/string/contains.rs index c21aeff71..6e1cbe6f9 100644 --- a/reductstore/src/storage/query/condition/value/string/contains.rs +++ b/reductstore/src/storage/query/condition/value/string/contains.rs @@ -20,8 +20,8 @@ pub(crate) trait Contains { impl Contains for Value { fn contains(self, other: Self) -> Result { - let other = other.as_string()?; - let self_string = self.as_string()?; + let other = other.to_string(); + let self_string = self.to_string(); Ok(self_string.contains(&other)) } } diff --git a/reductstore/src/storage/query/condition/value/string/ends_with.rs b/reductstore/src/storage/query/condition/value/string/ends_with.rs index dedf82320..8b3792fcf 100644 --- a/reductstore/src/storage/query/condition/value/string/ends_with.rs +++ b/reductstore/src/storage/query/condition/value/string/ends_with.rs @@ -20,8 +20,8 @@ pub(crate) trait EndsWith { impl EndsWith for Value { fn ends_with(self, other: Self) -> Result { - let other = other.as_string()?; - let self_string = self.as_string()?; + let other = other.to_string(); + let self_string = self.to_string(); Ok(self_string.ends_with(&other)) } } diff --git a/reductstore/src/storage/query/condition/value/string/starts_with.rs b/reductstore/src/storage/query/condition/value/string/starts_with.rs index 10faa338b..3b80aa538 100644 --- a/reductstore/src/storage/query/condition/value/string/starts_with.rs +++ b/reductstore/src/storage/query/condition/value/string/starts_with.rs @@ -20,8 +20,8 @@ pub(crate) trait StartsWith { impl StartsWith for Value { fn starts_with(self, other: Self) -> Result { - let other = other.as_string()?; - let self_string = self.as_string()?; + let other = other.to_string(); + let self_string = self.to_string(); Ok(self_string.starts_with(&other)) } } From b53a0409c0f61bca1fe8f2999b344b0ce1192d93 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 25 Jun 2025 18:03:42 +0200 Subject: [PATCH 44/93] Fix replication of update transaction to empty bucket (#867) * write record if update request failed with not found error * update CHANGELOG * clean code * remove check for old instances --- CHANGELOG.md | 5 ++ .../remote_bucket/client_wrapper.rs | 67 +++++++++---------- .../remote_bucket/states/bucket_available.rs | 46 +++++++++++-- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b0d21941..01045e57d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + +### Fixed + +- Fix replication of update transaction to empty bucket, [PR-867](https://github.com/reductstore/reductstore/pull/867) + ## [1.15.5] - 2025-06-10 ### Fixed diff --git a/reductstore/src/replication/remote_bucket/client_wrapper.rs b/reductstore/src/replication/remote_bucket/client_wrapper.rs index ba58e4413..eeef675fb 100644 --- a/reductstore/src/replication/remote_bucket/client_wrapper.rs +++ b/reductstore/src/replication/remote_bucket/client_wrapper.rs @@ -174,33 +174,30 @@ impl BucketWrapper { response: Result, ) -> Result, ReductError> { let mut failed_records = BTreeMap::new(); - if response.is_err() { - check_response(response)?; - } else { - let response = response.unwrap(); - let headers = response - .headers() - .iter() - .filter(|(key, _)| key.as_str().starts_with("x-reduct-error")) - .collect::>(); - - for (key, value) in headers { - let record_ts = key - .as_str() - .trim_start_matches("x-reduct-error-") - .parse::() - .map_err(|err| unprocessable_entity!("Invalid timestamp {}: {}", key, err))?; - - let (status, message) = value.to_str().unwrap().split_once(',').unwrap(); - failed_records.insert( - record_ts, - ReductError::new( - ErrorCode::from_int(status.parse().unwrap()).unwrap(), - message, - ), - ); - } + let response = check_response(response)?; + let headers = response + .headers() + .iter() + .filter(|(key, _)| key.as_str().starts_with("x-reduct-error-")) + .collect::>(); + + for (key, value) in headers { + let record_ts = key + .as_str() + .trim_start_matches("x-reduct-error-") + .parse::() + .map_err(|err| unprocessable_entity!("Invalid timestamp {}: {}", key, err))?; + + let (status, message) = value.to_str().unwrap().split_once(',').unwrap(); + failed_records.insert( + record_ts, + ReductError::new( + ErrorCode::from_int(status.parse().unwrap()).unwrap(), + message, + ), + ); } + Ok(failed_records) } @@ -209,7 +206,7 @@ impl BucketWrapper { } } -fn check_response(response: Result) -> Result<(), ReductError> { +fn check_response(response: Result) -> Result { let map_error = |error: reqwest::Error| -> ReductError { let status = if error.is_connect() { ErrorCode::ConnectionError @@ -226,7 +223,7 @@ fn check_response(response: Result) -> Result<(), ReductError> let response = response.map_err(map_error)?; if response.status().is_success() { - return Ok(()); + return Ok(response); } let status = @@ -296,8 +293,7 @@ impl ReductBucketApi for BucketWrapper { tx.send(response).await.unwrap(); }); - let failed_records = Self::parse_record_errors(rx.blocking_recv().unwrap())?; - Ok(failed_records) + Self::parse_record_errors(rx.blocking_recv().unwrap()) } fn update_batch( @@ -319,9 +315,7 @@ impl ReductBucketApi for BucketWrapper { tx.send(response).await.unwrap(); }); - let failed_records = Self::parse_record_errors(rx.blocking_recv().unwrap())?; - - Ok(failed_records) + Self::parse_record_errors(rx.blocking_recv().unwrap()) } fn server_url(&self) -> &str { @@ -450,7 +444,7 @@ mod tests { .body(Bytes::new()) .unwrap(); let response = Ok(response).map(|r| r.into()); - assert!(check_response(response).is_ok()); + assert_eq!(check_response(response).unwrap().status(), 200); } #[rstest] @@ -461,7 +455,10 @@ mod tests { .body(Bytes::new()) .unwrap(); let response = Ok(response).map(|r| r.into()); - assert!(check_response(response).is_err()); + assert_eq!( + check_response(response).unwrap_err().status(), + ErrorCode::NotFound + ); } } diff --git a/reductstore/src/replication/remote_bucket/states/bucket_available.rs b/reductstore/src/replication/remote_bucket/states/bucket_available.rs index 06a009bb8..fb3eef60c 100644 --- a/reductstore/src/replication/remote_bucket/states/bucket_available.rs +++ b/reductstore/src/replication/remote_bucket/states/bucket_available.rs @@ -1,4 +1,4 @@ -// Copyright 2023-2024 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 use crate::replication::remote_bucket::client_wrapper::{BoxedBucketApi, BoxedClientApi}; @@ -8,7 +8,6 @@ use crate::replication::remote_bucket::ErrorRecordMap; use crate::replication::Transaction; use crate::storage::entry::RecordReader; use log::{debug, warn}; -use reduct_base::error::ErrorCode::MethodNotAllowed; use reduct_base::error::{ErrorCode, ReductError}; use reduct_base::io::RecordMeta; use std::collections::BTreeMap; @@ -91,12 +90,19 @@ impl RemoteBucketState for BucketAvailableState { err ); - if err.status != MethodNotAllowed { - return self.check_error_and_change_state(err); + match err.status { + ErrorCode::NotFound => { + warn!( + "Entry {} not found on remote bucket {}/{}: {}", + entry_name, + self.bucket.server_url(), + self.bucket.name(), + err + ); + } + _ => return self.check_error_and_change_state(err), } - warn!("Please update the remote instance up to 1.11: {}", err); - let mut error_map = BTreeMap::new(); for record in &records_to_update { error_map.insert(record.timestamp(), err.clone()); @@ -343,11 +349,37 @@ mod tests { let state = state.write_batch("test", vec![record_to_update, record_to_write()]); assert!( state.last_result().is_ok(), - "we should not have any errors because wrote error records" + "we should not have any errors because wrote errored records" ); assert!(state.is_available()); } + #[rstest] + #[tokio::test] + async fn test_update_record_entry_not_found( + client: MockReductClientApi, + mut bucket: MockReductBucketApi, + record_to_update: (RecordReader, Transaction), + ) { + bucket + .expect_update_batch() + .returning(|_, _| Err(ReductError::new(ErrorCode::NotFound, "Entry not found"))); + + bucket.expect_write_batch().returning(|_, records| { + assert_eq!(records.len(), 1, "we create an entry and write the records"); + Ok(ErrorRecordMap::new()) + }); + + let state = Box::new(BucketAvailableState::new( + Box::new(client), + Box::new(bucket), + )); + + let state = state.write_batch("test", vec![record_to_update]); + assert!(state.last_result().is_ok()); + assert!(state.is_available()); + } + #[fixture] fn record_to_write() -> (RecordReader, Transaction) { let (_, rx) = tokio::sync::mpsc::channel(1); From 7249863e36072dfd895e3c716b90ee16c57c06c6 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 25 Jun 2025 18:05:28 +0200 Subject: [PATCH 45/93] release v1.15.6 --- CHANGELOG.md | 7 ++++++- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01045e57d..fbb7a7b7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.15.6] - 2025-06-25 ### Fixed @@ -1046,7 +1047,11 @@ reduct-rs: `ReductClient.url`, `ReductClient.token`, `ReductCientBuilder.try_bui - Initial release with basic HTTP API and FIFO bucket quota -[Unreleased]: https://github.com/reductstore/reductstore/compare/v1.15.4...HEAD +[Unreleased]: https://github.com/reductstore/reductstore/compare/v1.15.6...HEAD + +[1.15.6]: https://github.com/reductstore/reductstore/compare/v1.15.5...v1.15.6 + +[1.15.5]: https://github.com/reductstore/reductstore/compare/v1.15.4...v1.15.5 [1.15.4]: https://github.com/reductstore/reductstore/compare/v1.15.3...v1.15.4 diff --git a/Cargo.lock b/Cargo.lock index 659467341..0a28eaf0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1995,7 +1995,7 @@ dependencies = [ [[package]] name = "reduct-base" -version = "1.15.5" +version = "1.15.6" dependencies = [ "async-trait", "bytes", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "reduct-macros" -version = "1.15.5" +version = "1.15.6" dependencies = [ "quote", "syn 2.0.101", @@ -2021,7 +2021,7 @@ dependencies = [ [[package]] name = "reductstore" -version = "1.15.5" +version = "1.15.6" dependencies = [ "assert_matches", "async-stream", diff --git a/Cargo.toml b/Cargo.toml index 96220768c..5d4bb9305 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.15.5" +version = "1.15.6" authors = ["Alexey Timin ", "ReductSoftware UG "] edition = "2021" rust-version = "1.85.0" From 89748af0a452782b819d0ff4ac47a001d84018d9 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 27 Jun 2025 12:55:39 +0200 Subject: [PATCH 46/93] Support for "#ctx_before" and "#ctx_after" directives, (#866) * refactor filter interface for buffering * implement #before * avoid cloning meta information * use the same filter in extension repo * fix tests * clean code * fix build * implement ctx_after * test after and before context * test directive parsing * notify only replication for the same bucket * improve test coverage * fix filter rectod test * update CHANGELOG * format * Update reductstore/src/storage/query/filters/when/ctx_before.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + integration_tests/api/bucket_api_test.py | 1 + reduct_base/src/io.rs | 15 +- reductstore/src/api/entry/read_batched.rs | 19 +- reductstore/src/api/entry/update_batched.rs | 6 +- reductstore/src/api/entry/update_single.rs | 7 +- reductstore/src/api/entry/write_batched.rs | 7 +- reductstore/src/api/entry/write_single.rs | 3 +- reductstore/src/ext/ext_repository.rs | 76 ++-- reductstore/src/ext/ext_repository/create.rs | 4 +- reductstore/src/ext/filter.rs | 148 ++----- reductstore/src/replication.rs | 7 +- .../src/replication/replication_repository.rs | 53 ++- .../src/replication/replication_task.rs | 72 +++- .../src/replication/transaction_filter.rs | 242 ++++++++---- reductstore/src/storage/query/condition.rs | 1 + .../src/storage/query/condition/parser.rs | 151 ++++++- reductstore/src/storage/query/filters.rs | 102 ++++- .../src/storage/query/filters/each_n.rs | 40 +- .../src/storage/query/filters/each_s.rs | 38 +- .../src/storage/query/filters/exclude.rs | 70 ++-- .../src/storage/query/filters/include.rs | 66 ++-- .../src/storage/query/filters/record_state.rs | 36 +- .../src/storage/query/filters/time_range.rs | 53 ++- reductstore/src/storage/query/filters/when.rs | 372 ++++++++++++++++-- .../storage/query/filters/when/ctx_after.rs | 66 ++++ .../storage/query/filters/when/ctx_before.rs | 105 +++++ reductstore/src/storage/query/historical.rs | 93 +++-- 28 files changed, 1386 insertions(+), 468 deletions(-) create mode 100644 reductstore/src/storage/query/filters/when/ctx_after.rs create mode 100644 reductstore/src/storage/query/filters/when/ctx_before.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdb11772..d186be28f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Integrate ReductRos v0.1.0, [PR-856](https://github.com/reductstore/reductstore/pull/856) - Filter buckets by read permission in server information, [PR-849](https://github.com/reductstore/reductstore/pull/849) - Support for duration literals, [PR-864](https://github.com/reductstore/reductstore/pull/864) +- Support for `#ctx_before` and `#ctx_after` directives, [PR-866](https://github.com/reductstore/reductstore/pull/866) ### Changed diff --git a/integration_tests/api/bucket_api_test.py b/integration_tests/api/bucket_api_test.py index 3e0676747..f8b36dc34 100644 --- a/integration_tests/api/bucket_api_test.py +++ b/integration_tests/api/bucket_api_test.py @@ -1,4 +1,5 @@ import json +from time import sleep from .conftest import requires_env, auth_headers diff --git a/reduct_base/src/io.rs b/reduct_base/src/io.rs index c8218ec31..45fc45737 100644 --- a/reduct_base/src/io.rs +++ b/reduct_base/src/io.rs @@ -2,6 +2,7 @@ use crate::error::ReductError; use crate::{internal_server_error, Labels}; use async_trait::async_trait; use bytes::Bytes; +use std::collections::HashMap; use std::time::Duration; use tokio::runtime::Handle; @@ -40,8 +41,11 @@ impl BuilderRecordMeta { self } - pub fn labels(mut self, labels: Labels) -> Self { - self.labels = labels; + pub fn labels(mut self, labels: HashMap) -> Self { + self.labels = labels + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); self } @@ -55,8 +59,11 @@ impl BuilderRecordMeta { self } - pub fn computed_labels(mut self, computed_labels: Labels) -> Self { - self.computed_labels = computed_labels; + pub fn computed_labels(mut self, computed_labels: HashMap) -> Self { + self.computed_labels = computed_labels + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); self } diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index e2e788cc1..7d87541d9 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -144,14 +144,17 @@ async fn fetch_and_response_batched_records( }; match reader { - Ok(reader) => { - { - let (name, value) = make_batch_header(&reader); - header_size += name.as_str().len() + value.to_str().unwrap().len() + 2; - body_size += reader.meta().content_length(); - headers.insert(name, value); + Ok(next_readers) => { + for reader in next_readers { + { + let (name, value) = make_batch_header(&reader); + header_size += name.as_str().len() + value.to_str().unwrap().len() + 2; + body_size += reader.meta().content_length(); + headers.insert(name, value); + } + + readers.push(reader); } - readers.push(reader); if header_size > io_settings.batch_max_metadata_size || body_size > io_settings.batch_max_size @@ -209,7 +212,7 @@ async fn next_record_reader( query_path: &str, recv_timeout: Duration, ext_repository: &BoxedManageExtensions, -) -> Option> { +) -> Option, ReductError>> { // we need to wait for the first record if let Ok(result) = timeout( recv_timeout, diff --git a/reductstore/src/api/entry/update_batched.rs b/reductstore/src/api/entry/update_batched.rs index 173d3bb37..7c28879c4 100644 --- a/reductstore/src/api/entry/update_batched.rs +++ b/reductstore/src/api/entry/update_batched.rs @@ -9,6 +9,7 @@ use axum::extract::{Path, State}; use axum_extra::headers::HeaderMap; use reduct_base::batch::{parse_batched_header, sort_headers_by_time}; +use reduct_base::io::RecordMeta; use reduct_base::Labels; use crate::api::entry::common::err_to_batched_header; @@ -81,7 +82,10 @@ pub(crate) async fn update_batched_records( replication_repo.notify(TransactionNotification { bucket: bucket_name.clone(), entry: entry_name.clone(), - labels: new_labels, + meta: RecordMeta::builder() + .timestamp(time) + .labels(new_labels) + .build(), event: Transaction::UpdateRecord(time), })?; } diff --git a/reductstore/src/api/entry/update_single.rs b/reductstore/src/api/entry/update_single.rs index 79cda488e..c271cd24c 100644 --- a/reductstore/src/api/entry/update_single.rs +++ b/reductstore/src/api/entry/update_single.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use axum::body::Body; use axum::extract::{Path, Query, State}; use axum_extra::headers::HeaderMap; - +use reduct_base::io::RecordMeta; use reduct_base::Labels; use crate::api::entry::common::parse_timestamp_from_query; @@ -81,7 +81,10 @@ pub(crate) async fn update_record( .notify(TransactionNotification { bucket: bucket.clone(), entry: entry_name.clone(), - labels: batched_result.get(&ts).unwrap().clone()?, + meta: RecordMeta::builder() + .timestamp(ts) + .labels(batched_result.get(&ts).unwrap().clone()?.clone()) + .build(), event: Transaction::UpdateRecord(ts), })?; diff --git a/reductstore/src/api/entry/write_batched.rs b/reductstore/src/api/entry/write_batched.rs index e617e8d19..c1de31a81 100644 --- a/reductstore/src/api/entry/write_batched.rs +++ b/reductstore/src/api/entry/write_batched.rs @@ -19,7 +19,7 @@ use crate::storage::storage::IO_OPERATION_TIMEOUT; use log::{debug, error}; use reduct_base::batch::{parse_batched_header, sort_headers_by_time, RecordHeader}; use reduct_base::error::ReductError; -use reduct_base::io::WriteRecord; +use reduct_base::io::{RecordMeta, WriteRecord}; use reduct_base::{bad_request, internal_server_error, unprocessable_entity}; use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; @@ -107,7 +107,10 @@ pub(crate) async fn write_batched_records( TransactionNotification { bucket: bucket_name.clone(), entry: entry_name.clone(), - labels: ctx.header.labels.clone(), + meta: RecordMeta::builder() + .timestamp(ctx.time) + .labels(ctx.header.labels.clone()) + .build(), event: Transaction::WriteRecord(ctx.time), }, )?; diff --git a/reductstore/src/api/entry/write_single.rs b/reductstore/src/api/entry/write_single.rs index 188ef1ace..bb341dd75 100644 --- a/reductstore/src/api/entry/write_single.rs +++ b/reductstore/src/api/entry/write_single.rs @@ -15,6 +15,7 @@ use crate::storage::storage::IO_OPERATION_TIMEOUT; use futures_util::StreamExt; use log::{debug, error}; use reduct_base::error::ReductError; +use reduct_base::io::RecordMeta; use reduct_base::{bad_request, unprocessable_entity, Labels}; use std::collections::HashMap; use std::sync::Arc; @@ -117,7 +118,7 @@ pub(crate) async fn write_record( .notify(TransactionNotification { bucket: bucket.clone(), entry: path.get("entry_name").unwrap().to_string(), - labels, + meta: RecordMeta::builder().timestamp(ts).labels(labels).build(), event: WriteRecord(ts), })?; Ok(()) diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index 598b468b4..ada1c73dc 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -5,14 +5,13 @@ mod create; mod load; use crate::asset::asset_manager::ManageStaticAsset; -use crate::ext::filter::ExtWhenFilter; use crate::storage::query::base::QueryOptions; use crate::storage::query::condition::Parser; use crate::storage::query::QueryRx; use async_trait::async_trait; use dlopen2::wrapper::{Container, WrapperApi}; use futures_util::StreamExt; -use reduct_base::error::ErrorCode::{Interrupt, NoContent}; +use reduct_base::error::ErrorCode::NoContent; use reduct_base::error::ReductError; use reduct_base::ext::{ BoxedCommiter, BoxedProcessor, BoxedReadRecord, BoxedRecordStream, ExtSettings, IoExtension, @@ -59,14 +58,14 @@ pub(crate) trait ManageExtensions { &self, query_id: u64, query_rx: Arc>, - ) -> Option>; + ) -> Option, ReductError>>; } pub type BoxedManageExtensions = Box; pub(crate) struct QueryContext { query: QueryOptions, - condition_filter: ExtWhenFilter, + condition_filter: Box + Send + Sync>, last_access: Instant, current_stream: Option>, @@ -113,7 +112,7 @@ impl ManageExtensions for ExtRepository { // check if the query has references to computed labels and no extension is found let condition = if let Some(condition) = ext_query.remove("when") { - let node = Parser::new().parse(&condition)?; + let node = Parser::new().parse(condition)?; Some(node) } else { None @@ -165,7 +164,12 @@ impl ManageExtensions for ExtRepository { } if let Some((processor, commiter, condition)) = controllers { - let condition_filter = ExtWhenFilter::new(condition, true); + let condition_filter = if let Some(condition) = condition { + let (node, directives) = condition; + Box::new(WhenFilter::try_new(node, directives, true)?) + } else { + DummyFilter::boxed() + }; query_map.insert(query_id, { QueryContext { @@ -186,7 +190,7 @@ impl ManageExtensions for ExtRepository { &self, query_id: u64, query_rx: Arc>, - ) -> Option> { + ) -> Option, ReductError>> { // TODO: The code is awkward, we need to refactor it // unfortunately stream! macro does not work here and crashes compiler let mut lock = self.query_map.write().await; @@ -198,7 +202,7 @@ impl ManageExtensions for ExtRepository { .await .recv() .await - .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)); + .map(|record| record.map(|r| vec![Box::new(r) as BoxedReadRecord])); if result.is_none() { // If no record is available, return a no content error to finish the query. @@ -222,19 +226,29 @@ impl ManageExtensions for ExtRepository { let record = result.unwrap(); - return match query.condition_filter.filter_record(record) { - Some(result) => match result { - Ok(record) => query.commiter.commit_record(record).await, - Err(e) => { - if e.status == Interrupt { - query.current_stream = None; - None - } else { - Some(Err(e)) + return match query.condition_filter.filter(record) { + Ok(Some(records)) => { + let mut commited_records = vec![]; + for record in records { + if let Some(rec) = query.commiter.commit_record(record).await { + match rec { + Ok(rec) => commited_records.push(rec), + Err(e) => return Some(Err(e)), + } } } - }, - None => None, + + if commited_records.is_empty() { + None + } else { + Some(Ok(commited_records)) + } + } + Ok(None) => { + query.current_stream = None; + None + } + Err(e) => Some(Err(e)), }; } else { // stream is empty, we need to process the next record @@ -251,13 +265,18 @@ impl ManageExtensions for ExtRepository { Err(e) => { return if e.status == NoContent { if let Some(last_record) = query.commiter.flush().await { - Some(last_record) + match last_record { + Ok(rec) => { + Some(Ok(vec![rec])) // return the last record if available + } + Err(e) => Some(Err(e)), + } } else { - Some(Err(e)) + Some(Err(e)) // return no content error if no last record } } else { Some(Err(e)) - } + }; } }; @@ -273,6 +292,8 @@ impl ManageExtensions for ExtRepository { } } +use crate::ext::filter::DummyFilter; +use crate::storage::query::filters::{RecordFilter, WhenFilter}; pub(crate) use create::create_ext_repository; #[cfg(test)] @@ -386,8 +407,7 @@ pub(super) mod tests { assert_eq!( query_context .condition_filter - .filter_record(Box::new(MockRecord::new("not-in-when", "val"))) - .unwrap() + .filter(Box::new(MockRecord::new("not-in-when", "val"))) .err() .unwrap(), not_found!("Reference '@label' not found"), @@ -642,12 +662,16 @@ pub(super) mod tests { "First run should be None (stupid implementation)" ); - let record = mocked_ext_repo + let mut records = mocked_ext_repo .fetch_and_process_record(1, query_rx.clone()) .await + .unwrap() .unwrap(); - assert_eq!(record.unwrap().read().await, None); + assert_eq!(records.len(), 1, "Should return one record"); + + let record = records.first_mut().unwrap(); + assert_eq!(record.read().await, None); assert_eq!( mocked_ext_repo diff --git a/reductstore/src/ext/ext_repository/create.rs b/reductstore/src/ext/ext_repository/create.rs index 22eedd6a1..29a5fff8d 100644 --- a/reductstore/src/ext/ext_repository/create.rs +++ b/reductstore/src/ext/ext_repository/create.rs @@ -56,13 +56,13 @@ pub fn create_ext_repository( &self, _query_id: u64, query_rx: Arc>, - ) -> Option> { + ) -> Option, ReductError>> { let result = query_rx .write() .await .recv() .await - .map(|record| record.map(|r| Box::new(r) as BoxedReadRecord)); + .map(|record| record.map(|r| vec![Box::new(r) as BoxedReadRecord])); if result.is_none() { // If no record is available, return a no content error to finish the query. diff --git a/reductstore/src/ext/filter.rs b/reductstore/src/ext/filter.rs index 0c1346723..a48db6da2 100644 --- a/reductstore/src/ext/filter.rs +++ b/reductstore/src/ext/filter.rs @@ -1,138 +1,70 @@ -// Copyright 2025 ReductSoftware UG -// Licensed under the Business Source License 1.1 - -use crate::storage::query::condition::{BoxedNode, Context}; +use crate::storage::proto::record::State::Finished; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; use reduct_base::error::ReductError; use reduct_base::ext::BoxedReadRecord; use std::collections::HashMap; -pub(super) struct ExtWhenFilter { - condition: Option, - strict: bool, +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 +pub(super) struct DummyFilter {} + +impl RecordFilter for DummyFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { + Ok(Some(vec![record])) + } } -/// This filter is used to filter records based on a condition. -/// -/// It is used in the `ext` module to filter records after they processed by extension, -/// and it puts computed labels into the context. -impl ExtWhenFilter { - pub fn new(condition: Option, strict: bool) -> Self { - ExtWhenFilter { condition, strict } +impl DummyFilter { + pub fn boxed() -> Box + Send + Sync> { + Box::new(DummyFilter {}) } +} - pub fn filter_record( - &mut self, - record: BoxedReadRecord, - ) -> Option> { - if self.condition.is_none() { - return Some(Ok(record)); - } +impl FilterRecord for BoxedReadRecord { + fn state(&self) -> i32 { + Finished as i32 + } - // filter with computed labels - match self.filter_with_computed(&record) { - Ok(true) => Some(Ok(record)), - Ok(false) => None, - Err(e) => { - if self.strict { - Some(Err(e)) - } else { - None - } - } - } + fn timestamp(&self) -> u64 { + self.meta().timestamp() } - fn filter_with_computed(&mut self, reader: &BoxedReadRecord) -> Result { - let meta = reader.meta(); - let labels = meta - .labels() - .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect::>(); + fn labels(&self) -> HashMap<&String, &String> { + self.meta().labels().iter().map(|(k, v)| (k, v)).collect() + } - let computed_labels = meta + fn computed_labels(&self) -> HashMap<&String, &String> { + self.meta() .computed_labels() .iter() - .map(|(k, v)| (k.as_str(), v.as_str())) - .collect::>(); - - let context = Context::new(meta.timestamp(), labels, computed_labels); - Ok(self - .condition - .as_mut() - .unwrap() - .apply(&context)? - .as_bool()?) + .map(|(k, v)| (k, v)) + .collect() } } #[cfg(test)] mod tests { use super::*; - use crate::ext::ext_repository::tests::{mocked_record, MockRecord}; - use crate::storage::query::condition::Parser; + use crate::ext::ext_repository::tests::mocked_record; use rstest::rstest; - use serde_json::json; #[rstest] - fn pass_status_if_condition_none(mocked_record: Box) { - let mut filter = ExtWhenFilter::new(None, false); - assert!(filter.filter_record(mocked_record).unwrap().is_ok()) - } - - #[rstest] - fn not_ready_if_condition_false(mocked_record: Box) { - let mut filter = ExtWhenFilter::new( - Some( - Parser::new() - .parse(&json!({"$and": [false, "@key1"]})) - .unwrap(), - ), - true, - ); - assert!(filter.filter_record(mocked_record).is_none()) - } - - #[rstest] - fn ready_if_condition_true(mocked_record: Box) { - let mut filter = ExtWhenFilter::new( - Some( - Parser::new() - .parse(&json!({"$and": [true, "@key1"]})) - .unwrap(), - ), - true, - ); - assert!(filter.filter_record(mocked_record).unwrap().is_ok()) - } - - #[rstest] - fn ready_with_error_strict(mocked_record: Box) { - let mut filter = ExtWhenFilter::new( - Some( - Parser::new() - .parse(&json!({"$and": [true, "@not-exit"]})) - .unwrap(), - ), - true, + fn test_dummy_filter(mocked_record: BoxedReadRecord) { + let mut filter = DummyFilter::boxed(); + let result = filter.filter(mocked_record).unwrap(); + assert_eq!( + result.unwrap().len(), + 1, + "Dummy filter should pass all records" ); - assert!(filter.filter_record(mocked_record).unwrap().is_err()) } #[rstest] - fn ready_without_error(mocked_record: Box) { - let mut filter = ExtWhenFilter::new( - Some( - Parser::new() - .parse(&json!({"$and": [true, "@not-exit"]})) - .unwrap(), - ), - false, - ); - assert!( - filter.filter_record(mocked_record).is_none(), - "ignore bad condition" - ) + fn test_filter_record(mocked_record: BoxedReadRecord) { + assert_eq!(mocked_record.state(), Finished as i32); + assert_eq!(mocked_record.timestamp(), 0); + assert!(mocked_record.labels().is_empty()); + assert!(!mocked_record.computed_labels().is_empty()); } } diff --git a/reductstore/src/replication.rs b/reductstore/src/replication.rs index 85179cbab..51a3b9c9f 100644 --- a/reductstore/src/replication.rs +++ b/reductstore/src/replication.rs @@ -1,16 +1,15 @@ // Copyright 2023-2024 ReductSoftware UG // Licensed under the Business Source License 1.1 -use std::sync::Arc; - use crate::cfg::replication::ReplicationConfig; use crate::replication::replication_task::ReplicationTask; use crate::storage::storage::Storage; use reduct_base::error::ReductError; +use reduct_base::io::RecordMeta; use reduct_base::msg::replication_api::{ FullReplicationInfo, ReplicationInfo, ReplicationSettings, }; -use reduct_base::Labels; +use std::sync::Arc; mod diagnostics; pub mod proto; @@ -74,7 +73,7 @@ impl TryFrom for Transaction { pub struct TransactionNotification { pub bucket: String, pub entry: String, - pub labels: Labels, + pub meta: RecordMeta, pub event: Transaction, } pub trait ManageReplications { diff --git a/reductstore/src/replication/replication_repository.rs b/reductstore/src/replication/replication_repository.rs index 69635bd8b..aeb501676 100644 --- a/reductstore/src/replication/replication_repository.rs +++ b/reductstore/src/replication/replication_repository.rs @@ -194,6 +194,10 @@ impl ManageReplications for ReplicationRepository { fn notify(&mut self, notification: TransactionNotification) -> Result<(), ReductError> { for (_, replication) in self.replications.iter_mut() { + if replication.settings().src_bucket != notification.bucket { + continue; // skip if the replication is not for the source bucket + } + let _ = replication.notify(notification.clone())?; } Ok(()) @@ -304,7 +308,7 @@ impl ReplicationRepository { // check syntax of when condition if let Some(when) = &settings.when { - if let Err(e) = Parser::new().parse(when) { + if let Err(e) = Parser::new().parse(when.clone()) { return Err(unprocessable_entity!( "Invalid replication condition: {}", e @@ -626,6 +630,7 @@ mod tests { mod get { use super::*; + use reduct_base::io::RecordMeta; #[rstest] fn test_get_replication(mut repo: ReplicationRepository, settings: ReplicationSettings) { @@ -635,7 +640,7 @@ mod tests { repl.notify(TransactionNotification { bucket: "bucket-1".to_string(), entry: "entry-1".to_string(), - labels: Labels::default(), + meta: RecordMeta::builder().build(), event: WriteRecord(0), }) .unwrap(); @@ -668,6 +673,50 @@ mod tests { } } + mod notify { + use super::*; + use reduct_base::io::RecordMeta; + + #[rstest] + fn test_notify_replication(mut repo: ReplicationRepository, settings: ReplicationSettings) { + repo.create_replication("test", settings.clone()).unwrap(); + + let notification = TransactionNotification { + bucket: "bucket-1".to_string(), + entry: "entry-1".to_string(), + meta: RecordMeta::builder().build(), + event: WriteRecord(0), + }; + + repo.notify(notification.clone()).unwrap(); + let repl = repo.get_replication("test").unwrap(); + assert_eq!(repl.info().pending_records, 1); + } + + #[rstest] + fn test_notify_wrong_bucket( + mut repo: ReplicationRepository, + settings: ReplicationSettings, + ) { + repo.create_replication("test", settings.clone()).unwrap(); + + let notification = TransactionNotification { + bucket: "bucket-2".to_string(), + entry: "entry-1".to_string(), + meta: RecordMeta::builder().build(), + event: WriteRecord(0), + }; + + repo.notify(notification).unwrap(); + let repl = repo.get_replication("test").unwrap(); + assert_eq!( + repl.info().pending_records, + 0, + "Should not notify replication for wrong bucket" + ); + } + } + mod from { use super::*; diff --git a/reductstore/src/replication/replication_task.rs b/reductstore/src/replication/replication_task.rs index d4306cf20..c3e2696b6 100644 --- a/reductstore/src/replication/replication_task.rs +++ b/reductstore/src/replication/replication_task.rs @@ -219,32 +219,31 @@ impl ReplicationTask { pub fn notify(&mut self, notification: TransactionNotification) -> Result<(), ReductError> { // We need to have a filter for each entry - { + let entry_name = notification.entry.clone(); + let notifications = { let mut lock = self.filter_map.write()?; if !lock.contains_key(¬ification.entry) { lock.insert( notification.entry.clone(), - TransactionFilter::new(self.name(), self.settings.clone()), + TransactionFilter::try_new(self.name(), self.settings.clone())?, ); } - let filter = lock.get_mut(¬ification.entry).unwrap(); - if !filter.filter(¬ification) { - return Ok(()); - } - } + let filter = lock.get_mut(&entry_name).unwrap(); + filter.filter(notification) + }; // NOTE: very important not to lock the log_map for too long // because it is used by the replication thread - if !self.log_map.read()?.contains_key(¬ification.entry) { + if !self.log_map.read()?.contains_key(&entry_name) { let mut map = self.log_map.write()?; map.insert( - notification.entry.clone(), + entry_name.clone(), RwLock::new(TransactionLog::try_load_or_create( Self::build_path_to_transaction_log( self.storage.data_path(), &self.settings.src_bucket, - ¬ification.entry, + &entry_name, &self.name, ), self.system_options.transaction_log_size, @@ -253,12 +252,15 @@ impl ReplicationTask { }; let log_map = self.log_map.read()?; - let log = log_map.get(¬ification.entry).unwrap(); + let log = log_map.get(&entry_name).unwrap(); - if let Some(_) = log.write()?.push_back(notification.event.clone())? { - error!("Transaction log is full, dropping the oldest transaction without replication"); + for notification in notifications.into_iter() { + if let Some(_) = log.write()?.push_back(notification.event)? { + error!( + "Transaction log is full, dropping the oldest transaction without replication" + ); + } } - Ok(()) } @@ -331,11 +333,11 @@ mod tests { use bytes::Bytes; + use crate::core::file_cache::FILE_CACHE; use mockall::mock; + use reduct_base::io::RecordMeta; use rstest::*; - use crate::core::file_cache::FILE_CACHE; - use crate::replication::remote_bucket::ErrorRecordMap; use crate::replication::Transaction; use crate::storage::entry::io::record_reader::RecordReader; @@ -372,6 +374,42 @@ mod tests { ); } + #[rstest] + fn test_add_new_entry( + mut remote_bucket: MockRmBucket, + mut notification: TransactionNotification, + settings: ReplicationSettings, + ) { + remote_bucket + .expect_write_batch() + .returning(|_, _| Ok(ErrorRecordMap::new())); + remote_bucket.expect_is_active().return_const(true); + let mut replication = build_replication(remote_bucket, settings.clone()); + + notification.entry = "new_entry".to_string(); + fs::create_dir_all( + replication + .storage + .data_path() + .join(settings.src_bucket) + .join("new_entry"), + ) + .unwrap(); + + replication.notify(notification).unwrap(); + sleep(Duration::from_millis(100)); + assert!(transaction_log_is_empty(&replication)); + assert_eq!( + replication.info(), + ReplicationInfo { + name: "test".to_string(), + is_active: true, + is_provisioned: false, + pending_records: 0, + } + ); + } + #[rstest] fn test_replication_ok_active( mut remote_bucket: MockRmBucket, @@ -608,7 +646,7 @@ mod tests { TransactionNotification { bucket: "src".to_string(), entry: "test1".to_string(), - labels: Labels::new(), + meta: RecordMeta::builder().timestamp(10).build(), event: Transaction::WriteRecord(10), } } diff --git a/reductstore/src/replication/transaction_filter.rs b/reductstore/src/replication/transaction_filter.rs index 61b47be00..d0ce99819 100644 --- a/reductstore/src/replication/transaction_filter.rs +++ b/reductstore/src/replication/transaction_filter.rs @@ -2,29 +2,44 @@ // Licensed under the Business Source License 1.1 use log::warn; -use reduct_base::io::RecordMeta; +use reduct_base::error::ReductError; use reduct_base::msg::replication_api::ReplicationSettings; +use std::collections::HashMap; use crate::replication::TransactionNotification; use crate::storage::query::condition::Parser; use crate::storage::query::filters::{ - EachNFilter, EachSecondFilter, ExcludeLabelFilter, IncludeLabelFilter, RecordFilter, WhenFilter, + apply_filters_recursively, EachNFilter, EachSecondFilter, ExcludeLabelFilter, FilterRecord, + IncludeLabelFilter, RecordFilter, WhenFilter, }; +type Filter = Box + Send + Sync>; /// Filter for transaction notifications. pub(super) struct TransactionFilter { bucket: String, entries: Vec, - query_filters: Vec>, + query_filters: Vec, } -impl Into for TransactionNotification { - fn into(self) -> RecordMeta { - RecordMeta::builder() - .timestamp(self.event.into_timestamp()) - .state(0) - .labels(self.labels) - .build() +impl FilterRecord for TransactionNotification { + fn timestamp(&self) -> u64 { + *self.event.timestamp() + } + + fn labels(&self) -> HashMap<&String, &String> { + self.meta.labels().iter().map(|(k, v)| (k, v)).collect() + } + + fn computed_labels(&self) -> HashMap<&String, &String> { + self.meta + .computed_labels() + .iter() + .map(|(k, v)| (k, v)) + .collect() + } + + fn state(&self) -> i32 { + self.meta.state() } } @@ -37,8 +52,8 @@ impl TransactionFilter { /// * `entries` - Entries to filter. Supports wildcards. If empty, all entries are matched. /// * `include` - Labels to include. All must match. If empty, all labels are matched. /// * `exclude` - Labels to exclude. Any must match. If empty, no labels are matched. - pub(super) fn new(name: &str, settings: ReplicationSettings) -> Self { - let mut query_filters: Vec> = vec![]; + pub(super) fn try_new(name: &str, settings: ReplicationSettings) -> Result { + let mut query_filters: Vec = vec![]; if !settings.include.is_empty() { query_filters.push(Box::new(IncludeLabelFilter::new(settings.include))); } @@ -56,9 +71,9 @@ impl TransactionFilter { } if let Some(when) = settings.when { - match Parser::new().parse(&when) { - Ok(condition) => { - query_filters.push(Box::new(WhenFilter::new(condition))); + match Parser::new().parse(when.clone()) { + Ok((condition, directives)) => { + query_filters.push(Box::new(WhenFilter::try_new(condition, directives, true)?)); } Err(err) => warn!( "Error parsing when condition in {} replication task: {}", @@ -67,11 +82,11 @@ impl TransactionFilter { } } - Self { + Ok(Self { bucket: settings.src_bucket, entries: settings.entries, query_filters, - } + }) } /// Filter a transaction notification. @@ -83,9 +98,12 @@ impl TransactionFilter { /// # Returns /// /// `true` if the notification matches the filter, `false` otherwise. - pub(super) fn filter(&mut self, notification: &TransactionNotification) -> bool { + pub(super) fn filter( + &mut self, + notification: TransactionNotification, + ) -> Vec { if notification.bucket != self.bucket { - return false; + return vec![]; } if !self.entries.is_empty() { @@ -103,58 +121,67 @@ impl TransactionFilter { } } if !found { - return false; + return vec![]; } } - // filter out notifications - for filter in self.query_filters.iter_mut() { - let meta: RecordMeta = notification.clone().into(); - match filter.filter(&meta) { - Ok(false) => return false, - Err(err) => { - warn!("Error filtering transaction notification: {}", err); - return false; - } - _ => {} + match apply_filters_recursively( + self.query_filters.as_mut_slice(), + vec![notification.clone()], + 0, + ) { + Ok(Some(notifications)) => notifications, + Ok(None) => { + warn!( + "Filtering interrupted in replication task '{}'", + self.bucket + ); + vec![] + } + Err(err) => { + warn!( + "Error applying filters in replication task '{}': {}", + self.bucket, err + ); + vec![] } } - - true } } #[cfg(test)] mod tests { + use crate::replication::Transaction; + use reduct_base::io::RecordMeta; use reduct_base::Labels; use rstest::*; - use crate::replication::Transaction; - use super::*; #[rstest] fn test_transaction_filter(notification: TransactionNotification) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), ..ReplicationSettings::default() }, - ); - assert!(filter.filter(¬ification)); + ) + .unwrap(); + assert_eq!(filter.filter(notification).len(), 1); } #[rstest] fn test_transaction_filter_bucket(notification: TransactionNotification) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "other".to_string(), ..ReplicationSettings::default() }, - ); - assert!(!filter.filter(¬ification)); + ) + .unwrap(); + assert_eq!(filter.filter(notification).len(), 0); } #[rstest] @@ -168,15 +195,18 @@ mod tests { #[case] expected: bool, notification: TransactionNotification, ) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), entries, ..ReplicationSettings::default() }, - ); - assert_eq!(filter.filter(¬ification), expected); + ) + .unwrap(); + + let filtered = filter.filter(notification); + assert_eq!(filtered.is_empty(), !expected); } #[rstest] @@ -191,15 +221,18 @@ mod tests { #[case] expected: bool, notification: TransactionNotification, ) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), include: Labels::from_iter(include), ..ReplicationSettings::default() }, - ); - assert_eq!(filter.filter(¬ification), expected); + ) + .unwrap(); + + let filtered = filter.filter(notification); + assert_eq!(filtered.is_empty(), !expected); } #[rstest] @@ -213,120 +246,136 @@ mod tests { #[case] expected: bool, notification: TransactionNotification, ) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), exclude: Labels::from_iter(exclude), ..ReplicationSettings::default() }, - ); - assert_eq!(filter.filter(¬ification), expected); + ) + .unwrap(); + + let filtered = filter.filter(notification); + assert_eq!(filtered.is_empty(), !expected); } #[rstest] fn test_transaction_filter_each_n(notification: TransactionNotification) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), each_n: Some(2), ..ReplicationSettings::default() }, - ); + ) + .unwrap(); - assert!(filter.filter(¬ification)); - assert!(!filter.filter(¬ification)); - assert!(filter.filter(¬ification)); + assert_eq!(filter.filter(notification.clone()).len(), 1); + assert_eq!(filter.filter(notification.clone()).len(), 0); + assert_eq!(filter.filter(notification).len(), 1); } #[rstest] fn test_transaction_filter_each_s(mut notification: TransactionNotification) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), each_s: Some(1.0), ..ReplicationSettings::default() }, - ); + ) + .unwrap(); - assert!(filter.filter(¬ification)); + assert_eq!(filter.filter(notification.clone()).len(), 1); notification.event = Transaction::WriteRecord(1); - assert!(!filter.filter(¬ification)); + assert_eq!(filter.filter(notification.clone()).len(), 0); notification.event = Transaction::WriteRecord(2); - assert!(!filter.filter(¬ification)); + assert_eq!(filter.filter(notification.clone()).len(), 0); notification.event = Transaction::WriteRecord(1000_002); - assert!(filter.filter(¬ification)); + assert_eq!(filter.filter(notification.clone()).len(), 1); } #[rstest] fn test_transaction_filter_when(notification: TransactionNotification) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), when: Some(serde_json::json!({"$eq": ["&x", "y"]})), ..ReplicationSettings::default() }, - ); + ) + .unwrap(); + + assert_eq!(filter.filter(notification.clone()).len(), 1); - assert!(filter.filter(¬ification)); - filter = TransactionFilter::new( + filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), when: Some(serde_json::json!({"$eq": ["&x", "z"]})), ..ReplicationSettings::default() }, - ); - assert!(!filter.filter(¬ification)); + ) + .unwrap(); + + assert_eq!(filter.filter(notification).len(), 0); } #[rstest] fn test_transaction_filter_when_non_strict(notification: TransactionNotification) { - let mut filter = TransactionFilter::new( + let mut filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), when: Some(serde_json::json!({"$eq": ["&NOT_EXIST", "y"]})), ..ReplicationSettings::default() }, + ) + .unwrap(); + + assert!( + filter.filter(notification).is_empty(), + "label doesn't exist but we consider it as false" ); + } + + #[rstest] + fn test_transaction_filter_interrupted(notification: TransactionNotification) { + let mut filter = TransactionFilter::try_new( + "test", + ReplicationSettings { + src_bucket: "bucket".to_string(), + when: Some(serde_json::json!({"$limit": 0})), + ..ReplicationSettings::default() + }, + ) + .unwrap(); assert!( - !filter.filter(¬ification), + filter.filter(notification).is_empty(), "label doesn't exist but we consider it as false" ); } #[rstest] fn test_transaction_filter_invalid_when() { - let filter = TransactionFilter::new( + let filter = TransactionFilter::try_new( "test", ReplicationSettings { src_bucket: "bucket".to_string(), when: Some(serde_json::json!({"$UNKNOWN_OP": ["&x", "y", "z"]})), ..ReplicationSettings::default() }, - ); + ) + .unwrap(); assert!(filter.query_filters.is_empty()); } - mod filter_point { - - use super::*; - - #[rstest] - fn test_filter_point(notification: TransactionNotification) { - let meta: RecordMeta = notification.clone().into(); - assert_eq!(meta.timestamp(), 0); - assert_eq!(meta.labels(), ¬ification.labels); - assert_eq!(meta.state(), 0); - } - } - #[fixture] fn notification() -> TransactionNotification { let labels = Labels::from_iter(vec![ @@ -336,8 +385,33 @@ mod tests { TransactionNotification { bucket: "bucket".to_string(), entry: "entry".to_string(), - labels, + meta: RecordMeta::builder() + .timestamp(0) + .labels(labels) + .computed_labels(Labels::default()) + .state(0) + .build(), event: Transaction::WriteRecord(0), } } + + mod filter_record_impl { + use super::*; + + #[rstest] + fn test_filter_record_impl(notification: TransactionNotification) { + let record: Box = Box::new(notification.clone()); + assert_eq!(record.timestamp(), *notification.event.timestamp()); + + for (key, value) in notification.meta.labels() { + assert_eq!(record.labels().get(key), Some(&value)); + } + + for (key, value) in notification.meta.computed_labels() { + assert_eq!(record.computed_labels().get(key), Some(&value)); + } + + assert_eq!(record.state(), notification.meta.state()); + } + } } diff --git a/reductstore/src/storage/query/condition.rs b/reductstore/src/storage/query/condition.rs index 4ce180267..838549d70 100644 --- a/reductstore/src/storage/query/condition.rs +++ b/reductstore/src/storage/query/condition.rs @@ -65,4 +65,5 @@ impl Debug for BoxedNode { } } +pub(crate) use parser::Directives; pub(crate) use parser::Parser; diff --git a/reductstore/src/storage/query/condition/parser.rs b/reductstore/src/storage/query/condition/parser.rs index 0b360a574..ed1984596 100644 --- a/reductstore/src/storage/query/condition/parser.rs +++ b/reductstore/src/storage/query/condition/parser.rs @@ -1,4 +1,4 @@ -// Copyright 2024 ReductSoftware UG +// Copyright 2024-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 use crate::storage::query::condition::computed_reference::ComputedReference; @@ -17,10 +17,14 @@ use crate::storage::query::condition::{Boxed, BoxedNode}; use reduct_base::error::ReductError; use reduct_base::unprocessable_entity; use serde_json::{Map, Number, Value as JsonValue}; +use std::collections::HashMap; /// Parses a JSON object into a condition tree. pub(crate) struct Parser {} +pub(crate) type Directives = HashMap; +static DIRECTIVES: [&str; 2] = ["#ctx_before", "#ctx_after"]; + impl Parser { /// Parses a JSON object into a condition tree. /// @@ -32,9 +36,59 @@ impl Parser { /// /// A boxed node representing the condition tree. /// The root node is a `StagedAllOf` that aggregates all expressions - pub fn parse(&self, json: &JsonValue) -> Result { - let expressions = Self::parse_recursively(json)?; - Ok(AllOf::boxed(expressions)?) + pub fn parse(&self, mut json: JsonValue) -> Result<(BoxedNode, Directives), ReductError> { + // parse directives if any + let directives = Self::parse_directives(&mut json)?; + let expressions = Self::parse_recursively(&json)?; + Ok((AllOf::boxed(expressions)?, directives)) + } + + fn parse_directives(json: &mut JsonValue) -> Result { + let mut directives = Directives::new(); + let mut keys_to_remove = vec![]; + for (key, value) in json.as_object().unwrap_or(&Map::new()).iter() { + if key.starts_with("#") { + if !DIRECTIVES.contains(&key.as_str()) { + return Err(unprocessable_entity!( + "Directive '{}' is not supported", + key + )); + } + + let value = match value { + JsonValue::Bool(value) => Value::Bool(*value), + JsonValue::Number(value) => { + if value.is_i64() || value.is_u64() { + Value::Int(value.as_i64().unwrap()) + } else { + Value::Float(value.as_f64().unwrap()) + } + } + JsonValue::String(value) => { + if let Ok(duration) = parse_duration(value) { + duration + } else { + Value::String(value.to_string()) + } + } + _ => { + return Err(unprocessable_entity!( + "Directive '{}' must be a primitive value", + key + )); + } + }; + + directives.insert(key.to_string(), value); + keys_to_remove.push(key.to_string()); + } + } + + // Remove directives from the original JSON object + for key in keys_to_remove { + json.as_object_mut().unwrap().remove(&key); + } + Ok(directives) } fn parse_recursively(json: &JsonValue) -> Result, ReductError> { @@ -206,7 +260,7 @@ mod tests { let json = json!({ "$and": [true, {"$gt": [20, 10]}] }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -215,7 +269,7 @@ mod tests { let json = json!({ "&label": {"$gt": 10} }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); let context = Context::new(0, HashMap::from_iter(vec![("label", "20")]), HashMap::new()); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -225,7 +279,7 @@ mod tests { let json = json!({ "$and": [1, -2] }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -234,7 +288,7 @@ mod tests { let json = json!({ "$and": [1.1, -2.2] }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -243,7 +297,7 @@ mod tests { let json = json!({ "$and": ["a","b"] }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -252,7 +306,7 @@ mod tests { let json = json!({ "$eq": ["1h", 3600_000_000u64] }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert!(node.apply(&context).unwrap().as_bool().unwrap()); } @@ -265,7 +319,7 @@ mod tests { ] } ); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); let context = Context::new( 0, HashMap::from_iter(vec![("label", "true")]), @@ -282,7 +336,7 @@ mod tests { 1 ] }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert_eq!(node.apply(&context).unwrap(), Value::Int(1)); } @@ -291,7 +345,7 @@ mod tests { let json = json!({ "$limit": 100 }); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert_eq!(node.apply(&context).unwrap(), Value::Int(1)); } @@ -300,7 +354,7 @@ mod tests { let json = json!( { "$xx": [true, true] }); - let result = parser.parse(&json); + let result = parser.parse(json); assert_eq!( result.err().unwrap().to_string(), "[UnprocessableEntity] Operator '$xx' not supported" @@ -315,7 +369,7 @@ mod tests { "x": "y" } } ); - let result = parser.parse(&json); + let result = parser.parse(json); assert_eq!( result.err().unwrap().to_string(), "[UnprocessableEntity] Object notation must have exactly one operator" @@ -329,7 +383,7 @@ mod tests { "$in": [10, 20] } }); - let result = parser.parse(&json); + let result = parser.parse(json); assert_eq!( result.err().unwrap().to_string(), "[UnprocessableEntity] Array type is not supported: [10,20]" @@ -343,7 +397,7 @@ mod tests { "$and": null } }); - let result = parser.parse(&json); + let result = parser.parse(json); assert_eq!( result.err().unwrap().to_string(), "[UnprocessableEntity] Null type is not supported: null" @@ -355,13 +409,72 @@ mod tests { let json = json!({ "and": [true, true] }); - let result = parser.parse(&json); + let result = parser.parse(json); assert_eq!( result.err().unwrap().to_string(), "[UnprocessableEntity] Operator 'and' must start with '$'" ); } + mod parse_directives { + use super::*; + use rstest::rstest; + use serde::Serialize; + + #[rstest] + fn test_parse_directives(parser: Parser) { + let json = json!({ + "#ctx_before": "1h", + "#ctx_after": "2h" + }); + let (_, directives) = parser.parse(json).unwrap(); + assert_eq!(directives.len(), 2); + assert_eq!(directives["#ctx_before"], Value::Duration(3600_000_000)); + assert_eq!(directives["#ctx_after"], Value::Duration(7200_000_000)); + } + + #[rstest] + #[case(true, Value::Bool(true))] + #[case(123, Value::Int(123))] + #[case(123.45, Value::Float(123.45))] + #[case("test", Value::String("test".to_string()))] + fn test_parse_directives_primitive_values( + parser: Parser, + #[case] value: T, + #[case] expected: Value, + ) { + let json = json!({ + "#ctx_before": value, + }); + let (_, directives) = parser.parse(json).unwrap(); + assert_eq!(directives["#ctx_before"], expected); + } + + #[rstest] + fn test_parse_directives_invalid_value(parser: Parser) { + let json = json!({ + "#ctx_before": [1, 2, 3] + }); + let result = parser.parse(json); + assert_eq!( + result.err().unwrap().to_string(), + "[UnprocessableEntity] Directive '#ctx_before' must be a primitive value" + ); + } + + #[rstest] + fn test_parse_invalid_directive(parser: Parser) { + let json = json!({ + "#invalid_directive": "value" + }); + let result = parser.parse(json); + assert_eq!( + result.err().unwrap().to_string(), + "[UnprocessableEntity] Directive '#invalid_directive' is not supported" + ); + } + } + mod parse_operators { use super::*; #[rstest] @@ -420,7 +533,7 @@ mod tests { operands )) .unwrap(); - let mut node = parser.parse(&json).unwrap(); + let (mut node, _) = parser.parse(json).unwrap(); assert!( node.apply(&context).unwrap().as_bool().unwrap(), "{}", diff --git a/reductstore/src/storage/query/filters.rs b/reductstore/src/storage/query/filters.rs index 053bff966..c102a9fe2 100644 --- a/reductstore/src/storage/query/filters.rs +++ b/reductstore/src/storage/query/filters.rs @@ -1,4 +1,4 @@ -// Copyright 2023-2024 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 pub(crate) mod each_n; @@ -9,28 +9,100 @@ pub(crate) mod record_state; pub(crate) mod time_range; pub(crate) mod when; +pub(crate) use each_n::EachNFilter; +pub(crate) use each_s::EachSecondFilter; +pub(crate) use exclude::ExcludeLabelFilter; +pub(crate) use include::IncludeLabelFilter; +pub(crate) use record_state::RecordStateFilter; +use reduct_base::error::ReductError; +use std::collections::HashMap; +pub(crate) use time_range::TimeRangeFilter; +pub(crate) use when::WhenFilter; + +/// A trait to access record metadata for filtering purposes with minimal overhead. +pub(crate) trait FilterRecord { + fn state(&self) -> i32; + + fn timestamp(&self) -> u64; + + fn labels(&self) -> HashMap<&String, &String>; + + fn computed_labels(&self) -> HashMap<&String, &String>; +} + /// Trait for record filters in queries. -pub trait RecordFilter { +pub(crate) trait RecordFilter { /// Filter the record by condition. /// /// # Arguments /// /// * `record` - The record metadata to filter. + /// * `ctx` - The context for the filter, which can be used to pass additional information. /// /// # Returns /// - /// * `Ok(true)` if the record passes the filter, `Ok(false)` otherwise. - /// * Err(`ReductError::Interrupt`) if the filter is interrupted - /// * `Err(ReductError)` if an error occurs during filtering. - fn filter(&mut self, record: &RecordMeta) -> Result; + /// * `Ok(Some(Vec<(R, Self::Ctx)>))` - returns a vector of records and their contexts if the record matches the filter. Empty vector if no records match. + /// * `Ok(None)` - if the filtering is interrupted e.g. $limit operator is reached. + /// * `Err(ReductError)` - if an error occurs during filtering. + fn filter(&mut self, record: R) -> Result>, ReductError>; } -pub(crate) use each_n::EachNFilter; -pub(crate) use each_s::EachSecondFilter; -pub(crate) use exclude::ExcludeLabelFilter; -pub(crate) use include::IncludeLabelFilter; -pub(crate) use record_state::RecordStateFilter; -use reduct_base::error::ReductError; -use reduct_base::io::RecordMeta; -pub(crate) use time_range::TimeRangeFilter; -pub(crate) use when::WhenFilter; +pub(crate) fn apply_filters_recursively( + filters: &mut [Box + Send + Sync>], + notifications: Vec, + index: usize, +) -> Result>, ReductError> { + if index == filters.len() { + return Ok(Some(notifications)); + } + + for notification in notifications { + match filters[index].filter(notification)? { + Some(notifications) => { + if !notifications.is_empty() { + return apply_filters_recursively(filters, notifications, index + 1); + } + } + + None => return Ok(None), + } + } + + Ok(Some(vec![])) +} + +#[cfg(test)] + +mod tests { + use super::*; + use reduct_base::io::RecordMeta; + + #[derive(Debug, Clone, PartialEq)] + pub(super) struct TestFilterRecord { + meta: RecordMeta, + } + + impl From for TestFilterRecord { + fn from(meta: RecordMeta) -> Self { + TestFilterRecord { meta } + } + } + + impl FilterRecord for TestFilterRecord { + fn state(&self) -> i32 { + self.meta.state() + } + + fn timestamp(&self) -> u64 { + self.meta.timestamp() + } + + fn labels(&self) -> HashMap<&String, &String> { + self.meta.labels().iter().collect() + } + + fn computed_labels(&self) -> HashMap<&String, &String> { + self.meta.computed_labels().iter().collect() + } + } +} diff --git a/reductstore/src/storage/query/filters/each_n.rs b/reductstore/src/storage/query/filters/each_n.rs index 5a7d6941e..cf59711b3 100644 --- a/reductstore/src/storage/query/filters/each_n.rs +++ b/reductstore/src/storage/query/filters/each_n.rs @@ -1,9 +1,8 @@ // Copyright 2023-2024 ReductSoftware UG // Licensed under the Business Source License 1.1 -use crate::storage::query::filters::RecordFilter; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; use reduct_base::error::ReductError; -use reduct_base::io::RecordMeta; /// Filter that passes every N-th record pub struct EachNFilter { @@ -20,33 +19,48 @@ impl EachNFilter { } } -impl RecordFilter for EachNFilter { - fn filter(&mut self, _record: &RecordMeta) -> Result { +impl RecordFilter for EachNFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { let ret = self.count % self.n == 0; self.count += 1; - Ok(ret) + if ret { + Ok(Some(vec![record])) + } else { + Ok(Some(vec![])) + } } } #[cfg(test)] mod tests { use super::*; - + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; use rstest::*; #[rstest] fn test_each_n_filter() { let mut filter = EachNFilter::new(2); - let meta = RecordMeta::builder().build(); + let record: TestFilterRecord = RecordMeta::builder().build().into(); - assert!(filter.filter(&meta).unwrap(), "First time should pass"); - assert!( - !filter.filter(&meta).unwrap(), + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record.clone()]), + "First time should pass" + ); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![]), "Second time should not pass" ); - assert!(filter.filter(&meta).unwrap(), "Third time should pass"); - assert!( - !filter.filter(&meta).unwrap(), + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record.clone()]), + "Third time should pass" + ); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![]), "Fourth time should not pass" ); } diff --git a/reductstore/src/storage/query/filters/each_s.rs b/reductstore/src/storage/query/filters/each_s.rs index a08c6a873..e89865486 100644 --- a/reductstore/src/storage/query/filters/each_s.rs +++ b/reductstore/src/storage/query/filters/each_s.rs @@ -1,9 +1,8 @@ // Copyright 2023-2024 ReductSoftware UG // Licensed under the Business Source License 1.1 -use crate::storage::query::filters::RecordFilter; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; use reduct_base::error::ReductError; -use reduct_base::io::RecordMeta; /// Filter that passes every N-th record pub struct EachSecondFilter { @@ -23,36 +22,47 @@ impl EachSecondFilter { } } -impl RecordFilter for EachSecondFilter { - fn filter(&mut self, record: &RecordMeta) -> Result { +impl RecordFilter for EachSecondFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { let ret = record.timestamp() as i64 - self.last_time >= (self.s * 1_000_000.0) as i64; if ret { self.last_time = record.timestamp().clone() as i64; } - Ok(ret) + if ret { + Ok(Some(vec![record])) + } else { + Ok(Some(vec![])) + } } } #[cfg(test)] mod tests { use super::*; - + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; use rstest::*; #[rstest] fn test_each_s_filter() { let mut filter = EachSecondFilter::new(2.0); - let meta = RecordMeta::builder().timestamp(1000_000).build(); + let record: TestFilterRecord = RecordMeta::builder().timestamp(1000_000).build().into(); - assert!(filter.filter(&meta).unwrap()); - assert!(!filter.filter(&meta).unwrap()); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record.clone()]), + ); + assert_eq!(filter.filter(record.clone()).unwrap(), Some(vec![]),); - let meta = RecordMeta::builder().timestamp(2000_000).build(); - assert!(!filter.filter(&meta).unwrap()); + let record: TestFilterRecord = RecordMeta::builder().timestamp(2000_000).build().into(); + assert_eq!(filter.filter(record).unwrap(), Some(vec![]),); - let meta = RecordMeta::builder().timestamp(3000_000).build(); - assert!(filter.filter(&meta).unwrap()); - assert!(!filter.filter(&meta).unwrap()); + let record: TestFilterRecord = RecordMeta::builder().timestamp(3000_000).build().into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record.clone()]), + ); + assert_eq!(filter.filter(record.clone()).unwrap(), Some(vec![]),); } } diff --git a/reductstore/src/storage/query/filters/exclude.rs b/reductstore/src/storage/query/filters/exclude.rs index 05840662d..84f5f2ded 100644 --- a/reductstore/src/storage/query/filters/exclude.rs +++ b/reductstore/src/storage/query/filters/exclude.rs @@ -2,10 +2,9 @@ // Licensed under the Business Source License 1.1 use reduct_base::error::ReductError; -use reduct_base::io::RecordMeta; use reduct_base::Labels; -use crate::storage::query::filters::RecordFilter; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; /// Filter that excludes records with specific labels pub struct ExcludeLabelFilter { @@ -27,21 +26,27 @@ impl ExcludeLabelFilter { } } -impl RecordFilter for ExcludeLabelFilter { - fn filter(&mut self, record: &RecordMeta) -> Result { - let result = !self - .labels - .iter() - .all(|(key, value)| record.labels().iter().any(|(k, v)| k == key && v == value)); - - Ok(result) +impl RecordFilter for ExcludeLabelFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { + let result = !self.labels.iter().all(|(key, value)| { + record + .labels() + .iter() + .any(|(k, v)| *k == key && *v == value) + }); + if result { + Ok(Some(vec![record])) + } else { + Ok(Some(vec![])) + } } } #[cfg(test)] mod tests { use super::*; - + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; use rstest::*; #[rstest] @@ -51,13 +56,19 @@ mod tests { "value".to_string(), )])); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![( "key".to_string(), "value".to_string(), )])) - .build(); - assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); + .build() + .into(); + + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![]), + "Record with key=value should not pass" + ); } #[rstest] @@ -67,13 +78,18 @@ mod tests { "value".to_string(), )])); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![( "key".to_string(), "other".to_string(), )])) - .build(); - assert!(filter.filter(&meta).unwrap(), "Record should pass"); + .build() + .into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record]), + "Record with key=other should pass" + ); } #[rstest] @@ -83,26 +99,30 @@ mod tests { ("key2".to_string(), "value2".to_string()), ])); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![ ("key1".to_string(), "value1".to_string()), ("key2".to_string(), "value2".to_string()), ("key3".to_string(), "value3".to_string()), ])) - .build(); - assert!( - !filter.filter(&meta).unwrap(), + .build() + .into(); + assert_eq!( + filter.filter(record).unwrap(), + Some(vec![]), "Record should not pass because it has key1=value1 and key2=value2" ); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![( "key1".to_string(), "value1".to_string(), )])) - .build(); - assert!( - filter.filter(&meta).unwrap(), + .build() + .into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record]), "Record should pass because it has only key1=value1" ); } diff --git a/reductstore/src/storage/query/filters/include.rs b/reductstore/src/storage/query/filters/include.rs index 93e8c9bf3..0438d0285 100644 --- a/reductstore/src/storage/query/filters/include.rs +++ b/reductstore/src/storage/query/filters/include.rs @@ -4,7 +4,7 @@ use reduct_base::error::ReductError; use reduct_base::Labels; -use crate::storage::query::filters::{RecordFilter, RecordMeta}; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; /// Filter that excludes records with specific labels pub struct IncludeLabelFilter { @@ -26,14 +26,20 @@ impl IncludeLabelFilter { } } -impl RecordFilter for IncludeLabelFilter { - fn filter(&mut self, record: &RecordMeta) -> Result { - let result = self - .labels - .iter() - .all(|(key, value)| record.labels().iter().any(|(k, v)| k == key && v == value)); +impl RecordFilter for IncludeLabelFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { + let result = self.labels.iter().all(|(key, value)| { + record + .labels() + .iter() + .any(|(k, v)| *k == key && *v == value) + }); - Ok(result) + if result { + Ok(Some(vec![record])) + } else { + Ok(Some(vec![])) + } } } @@ -41,6 +47,8 @@ impl RecordFilter for IncludeLabelFilter { mod tests { use super::*; + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; use rstest::*; #[rstest] @@ -50,13 +58,18 @@ mod tests { "value".to_string(), )])); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![( "key".to_string(), "value".to_string(), )])) - .build(); - assert!(filter.filter(&meta).unwrap(), "Record should pass"); + .build() + .into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record]), + "Record should pass" + ); } #[rstest] @@ -65,13 +78,18 @@ mod tests { "key".to_string(), "value".to_string(), )])); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![( "key".to_string(), "other".to_string(), )])) - .build(); - assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); + .build() + .into(); + assert_eq!( + filter.filter(record).unwrap(), + Some(vec![]), + "Record should not pass" + ); } #[rstest] @@ -81,26 +99,30 @@ mod tests { ("key2".to_string(), "value2".to_string()), ])); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![ ("key1".to_string(), "value1".to_string()), ("key2".to_string(), "value2".to_string()), ("key3".to_string(), "value3".to_string()), ])) - .build(); - assert!( - filter.filter(&meta).unwrap(), + .build() + .into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record]), "Record should pass because it has key1=value1 and key2=value2" ); - let meta = RecordMeta::builder() + let record: TestFilterRecord = RecordMeta::builder() .labels(Labels::from_iter(vec![( "key1".to_string(), "value1".to_string(), )])) - .build(); - assert!( - !filter.filter(&meta).unwrap(), + .build() + .into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![]), "Record should not pass because it has only key1=value1" ); } diff --git a/reductstore/src/storage/query/filters/record_state.rs b/reductstore/src/storage/query/filters/record_state.rs index 2b933568c..9b8c039dd 100644 --- a/reductstore/src/storage/query/filters/record_state.rs +++ b/reductstore/src/storage/query/filters/record_state.rs @@ -4,7 +4,7 @@ use crate::storage::proto::record::State; use reduct_base::error::ReductError; -use crate::storage::query::filters::{RecordFilter, RecordMeta}; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; /// Filter that passes records with a specific state pub struct RecordStateFilter { @@ -26,10 +26,14 @@ impl RecordStateFilter { } } -impl RecordFilter for RecordStateFilter { - fn filter(&mut self, record: &RecordMeta) -> Result { +impl RecordFilter for RecordStateFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { let result = record.state() == self.state as i32; - Ok(result) + if result { + Ok(Some(vec![record])) + } else { + Ok(Some(vec![])) + } } } @@ -38,20 +42,36 @@ mod tests { use super::*; use crate::storage::proto::record::State; + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; use rstest::*; #[rstest] fn test_record_state_filter() { let mut filter = RecordStateFilter::new(State::Finished); - let meta = RecordMeta::builder().state(State::Finished as i32).build(); - assert!(filter.filter(&meta).unwrap(), "Record should pass"); + let record: TestFilterRecord = RecordMeta::builder() + .state(State::Finished as i32) + .build() + .into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record]), + "Record should pass" + ); } #[rstest] fn test_record_state_filter_no_records() { let mut filter = RecordStateFilter::new(State::Finished); - let meta = RecordMeta::builder().state(State::Started as i32).build(); - assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); + let record: TestFilterRecord = RecordMeta::builder() + .state(State::Started as i32) + .build() + .into(); + assert_eq!( + filter.filter(record).unwrap(), + Some(vec![]), + "Record should not pass" + ); } } diff --git a/reductstore/src/storage/query/filters/time_range.rs b/reductstore/src/storage/query/filters/time_range.rs index 7058c166a..0775da5b5 100644 --- a/reductstore/src/storage/query/filters/time_range.rs +++ b/reductstore/src/storage/query/filters/time_range.rs @@ -1,7 +1,7 @@ // Copyright 2023-2024 ReductSoftware UG // Licensed under the Business Source License 1.1 -use crate::storage::query::filters::{RecordFilter, RecordMeta}; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; use reduct_base::error::ReductError; /// Filter that passes records with a timestamp within a specific range @@ -26,16 +26,20 @@ impl TimeRangeFilter { } } -impl RecordFilter for TimeRangeFilter { - fn filter(&mut self, record: &RecordMeta) -> Result { - let ts = record.timestamp() as u64; +impl RecordFilter for TimeRangeFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { + let ts = record.timestamp(); let ret = ts >= self.start && ts < self.stop; if ret { // Ensure that we don't return the same record twice self.start = ts + 1; } - Ok(ret) + if ret { + Ok(Some(vec![record])) + } else { + Ok(Some(vec![])) + } } } @@ -43,15 +47,22 @@ impl RecordFilter for TimeRangeFilter { mod tests { use super::*; + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; use rstest::*; #[rstest] fn test_time_range_filter() { let mut filter = TimeRangeFilter::new(0, 10); - let meta = RecordMeta::builder().timestamp(5).build(); - assert!(filter.filter(&meta).unwrap(), "First time should pass"); - assert!( - !filter.filter(&meta).unwrap(), + let record: TestFilterRecord = RecordMeta::builder().timestamp(5).build().into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record.clone()]), + "First time should pass" + ); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![]), "Second time should not pass, as we have already returned the record" ); } @@ -59,23 +70,35 @@ mod tests { #[rstest] fn test_time_range_filter_no_records() { let mut filter = TimeRangeFilter::new(0, 10); - let meta = RecordMeta::builder().timestamp(15).build(); - assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); + let record: TestFilterRecord = RecordMeta::builder().timestamp(15).build().into(); + assert_eq!( + filter.filter(record).unwrap(), + Some(vec![]), + "Record should not pass" + ); } #[rstest] fn test_time_include_start() { let mut filter = TimeRangeFilter::new(0, 10); - let meta = RecordMeta::builder().timestamp(0).build(); - assert!(filter.filter(&meta).unwrap(), "Record should pass"); + let record: TestFilterRecord = RecordMeta::builder().timestamp(0).build().into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![record]), + "Record should pass" + ); } #[rstest] fn test_time_exclude_end() { let mut filter = TimeRangeFilter::new(0, 10); - let meta = RecordMeta::builder().timestamp(10).build(); - assert!(!filter.filter(&meta).unwrap(), "Record should not pass"); + let record: TestFilterRecord = RecordMeta::builder().timestamp(10).build().into(); + assert_eq!( + filter.filter(record.clone()).unwrap(), + Some(vec![]), + "Record should not pass" + ); } } diff --git a/reductstore/src/storage/query/filters/when.rs b/reductstore/src/storage/query/filters/when.rs index d885ddb44..50ec95ed7 100644 --- a/reductstore/src/storage/query/filters/when.rs +++ b/reductstore/src/storage/query/filters/when.rs @@ -1,23 +1,88 @@ -// Copyright 2023-2024 ReductSoftware UG +// Copyright 2024-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 -use crate::storage::query::condition::{BoxedNode, Context}; -use crate::storage::query::filters::{RecordFilter, RecordMeta}; -use reduct_base::error::ReductError; +mod ctx_after; +mod ctx_before; + +use crate::storage::query::condition::{BoxedNode, Context, Directives}; +use crate::storage::query::filters::when::ctx_after::CtxAfter; +use crate::storage::query::filters::when::ctx_before::CtxBefore; +use crate::storage::query::filters::when::Padding::{Duration, Records}; +use crate::storage::query::filters::{FilterRecord, RecordFilter}; +use reduct_base::error::{ErrorCode, ReductError}; +use std::collections::VecDeque; + +pub(super) enum Padding { + Records(usize), + Duration(u64), +} /// A node representing a when filter with a condition. -pub struct WhenFilter { +pub struct WhenFilter { condition: BoxedNode, + strict: bool, + + ctx_before: CtxBefore, + ctx_after: CtxAfter, + ctx_buffer: VecDeque, } -impl WhenFilter { - pub fn new(condition: BoxedNode) -> Self { - WhenFilter { condition } +impl WhenFilter { + pub fn try_new( + condition: BoxedNode, + directives: Directives, + strict: bool, + ) -> Result { + let before = if let Some(before) = directives.get("#ctx_before") { + let val = before.as_int()?; + if val < 0 { + return Err(ReductError::unprocessable_entity( + "#ctx_before must be non-negative", + )); + } + + if before.is_duration() { + Duration(val as u64) + } else { + Records(val as usize) + } + } else { + Records(0) // Default to 0 records before + }; + + let after = if let Some(after) = directives.get("#ctx_after") { + let val = after.as_int()?; + if val < 0 { + return Err(ReductError::unprocessable_entity( + "#ctx_after must be non-negative", + )); + } + if after.is_duration() { + Duration(val as u64) + } else { + Records(val as usize) + } + } else { + Records(0) // Default to 0 records after + }; + + Ok(Self { + condition, + strict, + ctx_before: CtxBefore::new(before), + ctx_after: CtxAfter::new(after), + ctx_buffer: VecDeque::new(), + }) } } -impl RecordFilter for WhenFilter { - fn filter(&mut self, record: &RecordMeta) -> Result { +impl RecordFilter for WhenFilter { + fn filter(&mut self, record: R) -> Result>, ReductError> { + self.ctx_before.queue_record(&mut self.ctx_buffer, record); + + let record = self.ctx_buffer.back().unwrap(); + + // Prepare the context for the condition evaluation let context = Context::new( record.timestamp(), record @@ -31,34 +96,283 @@ impl RecordFilter for WhenFilter { .map(|(k, v)| (k.as_str(), v.as_str())) .collect(), ); - Ok(self.condition.apply(&context)?.as_bool()?) + + let result = match self.condition.apply(&context) { + Ok(value) => value.as_bool()?, + Err(err) => { + if err.status == ErrorCode::Interrupt { + return Ok(None); + } + + if self.strict { + // in strict mode, we return an error if a filter fails + return Err(err); + } + + false + } + }; + + if self.ctx_after.check(result, record.timestamp()) { + Ok(Some(self.ctx_buffer.drain(..).collect())) + } else { + Ok(Some(vec![])) + } } } #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; use crate::storage::query::condition::Parser; - use reduct_base::Labels; - use rstest::rstest; + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; + use rstest::{fixture, rstest}; + use serde_json::json; #[rstest] - fn filter() { - let parser = Parser::new(); - let json = serde_json::from_str(r#"{"$and": [true, "&label"]}"#).unwrap(); - let condition = parser.parse(&json).unwrap(); - - let mut filter = WhenFilter::new(condition); - - let meta = RecordMeta::builder() - .timestamp(0) - .labels(Labels::from_iter(vec![( - "label".to_string(), - "true".to_string(), - )])) - .build(); - let result = filter.filter(&meta).unwrap(); - assert_eq!(result, true); + fn filter(parser: Parser, record_true: TestFilterRecord) { + let (condition, directives) = parser + .parse(json!({ + "$and": [true, "&label"] + })) + .unwrap(); + + let mut filter = WhenFilter::try_new(condition, directives, true).unwrap(); + + let result = filter.filter(record_true.clone()).unwrap(); + assert_eq!(result, Some(vec![record_true])); + } + + mod context_n { + use super::*; + + #[rstest] + fn filter_ctx_before_n( + parser: Parser, + record_true: TestFilterRecord, + record_false: TestFilterRecord, + ) { + let (condition, directives) = parser + .parse(json!({ + "#ctx_before": 2, + "$and": [true, "&label"] + })) + .unwrap(); + + let mut filter = WhenFilter::try_new(condition, directives, true).unwrap(); + + let result = filter.filter(record_false.clone()).unwrap(); + assert_eq!(result, Some(vec![])); + + let result = filter.filter(record_false.clone()).unwrap(); + assert_eq!(result, Some(vec![])); + + let result = filter.filter(record_false.clone()).unwrap(); + assert_eq!(result, Some(vec![])); + + let result = filter.filter(record_true.clone()).unwrap(); + assert_eq!( + result, + Some(vec![ + record_false.clone(), + record_false, + record_true.clone() + ]) + ); + + let result = filter.filter(record_true.clone()).unwrap(); + assert_eq!(result, Some(vec![record_true])); + } + + #[rstest] + fn filter_ctx_after_n( + parser: Parser, + record_true: TestFilterRecord, + record_false: TestFilterRecord, + ) { + let (condition, directives) = parser + .parse(json!({ + "#ctx_after": 2, + "$and": [true, "&label"] + })) + .unwrap(); + + let mut filter = WhenFilter::try_new(condition, directives, true).unwrap(); + + let result = filter.filter(record_true.clone()).unwrap(); + assert_eq!(result, Some(vec![record_true.clone()])); + + let result = filter.filter(record_false.clone()).unwrap(); + assert_eq!(result, Some(vec![record_false.clone()])); + + let result = filter.filter(record_false.clone()).unwrap(); + assert_eq!(result, Some(vec![record_false])); + + let result = filter.filter(record_true.clone()).unwrap(); + assert_eq!(result, Some(vec![record_true])); + } + + #[rstest] + fn filter_ctx_before_negative(parser: Parser) { + let (condition, directives) = parser + .parse(json!({ + "#ctx_before": -2, + "$and": [true, "&label"] + })) + .unwrap(); + + let err = WhenFilter::::try_new(condition, directives, true) + .err() + .unwrap(); + assert_eq!( + err, + ReductError::unprocessable_entity("#ctx_before must be non-negative") + ); + } + + #[rstest] + fn filter_ctx_after_negative(parser: Parser) { + let (condition, directives) = parser + .parse(json!({ + "#ctx_after": -2, + "$and": [true, "&label"] + })) + .unwrap(); + + let err = WhenFilter::::try_new(condition, directives, true) + .err() + .unwrap(); + assert_eq!( + err, + ReductError::unprocessable_entity("#ctx_after must be non-negative") + ); + } + + #[fixture] + fn record_false() -> TestFilterRecord { + RecordMeta::builder() + .labels(HashMap::from_iter(vec![("label", "false")])) + .build() + .into() + } + } + + mod context_dur { + use super::*; + + #[rstest] + fn filter_ctx_before_duration( + parser: Parser, + record_false_3: TestFilterRecord, + record_false_4: TestFilterRecord, + record_true_5: TestFilterRecord, + ) { + let (condition, directives) = parser + .parse(json!({ + "#ctx_before": "2ms", + "$and": [true, "&label"] + })) + .unwrap(); + + let mut filter = WhenFilter::try_new(condition, directives, true).unwrap(); + let result = filter.filter(record_false_3.clone()).unwrap(); + assert_eq!(result, Some(vec![])); + + let result = filter.filter(record_false_4.clone()).unwrap(); + assert_eq!(result, Some(vec![])); + + let result = filter.filter(record_true_5.clone()).unwrap(); + assert_eq!( + result, + Some(vec![record_false_3, record_false_4, record_true_5]) + ); + } + + #[rstest] + fn filter_ctx_after_duration( + parser: Parser, + record_true_5: TestFilterRecord, + record_false_6: TestFilterRecord, + record_false_7: TestFilterRecord, + ) { + let (condition, directives) = parser + .parse(json!({ + "#ctx_after": "2ms", + "$and": [true, "&label"] + })) + .unwrap(); + + let mut filter = WhenFilter::try_new(condition, directives, true).unwrap(); + + let result = filter.filter(record_true_5.clone()).unwrap(); + assert_eq!(result, Some(vec![record_true_5])); + + let result = filter.filter(record_false_6.clone()).unwrap(); + assert_eq!(result, Some(vec![record_false_6])); + + let result = filter.filter(record_false_7.clone()).unwrap(); + assert_eq!(result, Some(vec![record_false_7])); + } + + #[fixture] + fn record_false_3() -> TestFilterRecord { + RecordMeta::builder() + .timestamp(3000) + .labels(HashMap::from_iter(vec![("label", "false")])) + .build() + .into() + } + + #[fixture] + fn record_false_7() -> TestFilterRecord { + RecordMeta::builder() + .timestamp(7000) + .labels(HashMap::from_iter(vec![("label", "false")])) + .build() + .into() + } + + #[fixture] + fn record_false_6() -> TestFilterRecord { + RecordMeta::builder() + .timestamp(6000) + .labels(HashMap::from_iter(vec![("label", "false")])) + .build() + .into() + } + + #[fixture] + fn record_true_5() -> TestFilterRecord { + RecordMeta::builder() + .timestamp(5000) + .labels(HashMap::from_iter(vec![("label", "true")])) + .build() + .into() + } + + #[fixture] + fn record_false_4() -> TestFilterRecord { + RecordMeta::builder() + .timestamp(4000) + .labels(HashMap::from_iter(vec![("label", "false")])) + .build() + .into() + } + } + + #[fixture] + fn parser() -> Parser { + Parser::new() + } + + #[fixture] + fn record_true() -> TestFilterRecord { + RecordMeta::builder() + .labels(HashMap::from_iter(vec![("label", "true")])) + .build() + .into() } } diff --git a/reductstore/src/storage/query/filters/when/ctx_after.rs b/reductstore/src/storage/query/filters/when/ctx_after.rs new file mode 100644 index 000000000..dc89294a3 --- /dev/null +++ b/reductstore/src/storage/query/filters/when/ctx_after.rs @@ -0,0 +1,66 @@ +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 + +use crate::storage::query::filters::when::Padding; +use crate::storage::query::filters::when::Padding::{Duration, Records}; + +/// Context for managing records after a condition is checked in a `when` filter. +pub(super) struct CtxAfter { + after: Padding, + count: i64, + last_ts: Option, +} + +impl CtxAfter { + pub fn new(after: Padding) -> Self { + CtxAfter { + after, + count: 0, + last_ts: None, + } + } + + /// Checks the condition against the context, updating the state based on the padding type. + pub(crate) fn check(&mut self, condition: bool, time: u64) -> bool { + match self.after { + Records(n) => { + self.count -= 1; + if condition { + self.count = n as i64; + } + self.count >= 0 + } + Duration(us) => { + if condition { + self.last_ts = Some(time); + } + + self.last_ts.is_some_and(|last_ts| last_ts + us >= time) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ctx_after_records() { + let mut ctx = CtxAfter::new(Records(3)); + assert!(ctx.check(true, 1000)); + assert!(ctx.check(false, 2000)); + assert!(ctx.check(false, 3000)); + assert!(ctx.check(false, 4000)); + assert!(!ctx.check(false, 5000)); // Should be false after 3 records + } + + #[test] + fn test_ctx_after_duration() { + let mut ctx = CtxAfter::new(Duration(5000)); + assert!(ctx.check(true, 1000)); + assert!(ctx.check(false, 2000)); + assert!(ctx.check(false, 6000)); // Should still be true due to duration + assert!(!ctx.check(false, 7000)); // Should be false after duration expires + } +} diff --git a/reductstore/src/storage/query/filters/when/ctx_before.rs b/reductstore/src/storage/query/filters/when/ctx_before.rs new file mode 100644 index 000000000..f2e58e533 --- /dev/null +++ b/reductstore/src/storage/query/filters/when/ctx_before.rs @@ -0,0 +1,105 @@ +// Copyright 2025 ReductSoftware UG +// Licensed under the Business Source License 1.1 + +use crate::storage::query::filters::when::Padding; +use crate::storage::query::filters::when::Padding::{Duration, Records}; +use crate::storage::query::filters::FilterRecord; +use std::collections::VecDeque; + +/// Context for managing records before a condition is checked in a `when` filter. +pub(super) struct CtxBefore { + before: Padding, +} + +impl CtxBefore { + pub fn new(before: Padding) -> Self { + CtxBefore { before } + } + + /// Queues a record into the context buffer, managing the size based on the `before` padding. + /// + /// Note: we need to keep the buffer outside of the filter to allow use reference to the first record + /// + /// # Arguments + /// + /// * `ctx_buffer` - A mutable reference to the buffer where records are stored. + /// * `record` - The record to be queued. + pub(crate) fn queue_record(&self, ctx_buffer: &mut VecDeque, record: R) + where + R: FilterRecord, + { + ctx_buffer.push_back(record); + match self.before { + Records(n) => { + if ctx_buffer.len() > n + 1 { + ctx_buffer.pop_front(); + } + } + Duration(us) => { + let mut first_record_ts = ctx_buffer.front().unwrap().timestamp(); + let last_record_ts = ctx_buffer.back().unwrap().timestamp(); + while last_record_ts - first_record_ts > us { + ctx_buffer.pop_front().unwrap(); + first_record_ts = ctx_buffer.front().map_or(0, |r| r.timestamp()); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::query::filters::tests::TestFilterRecord; + use reduct_base::io::RecordMeta; + use rstest::*; + + #[rstest] + fn test_ctx_before_records() { + let ctx = CtxBefore::new(Records(2)); + let mut buffer = VecDeque::new(); + let record: TestFilterRecord = RecordMeta::builder().build().into(); + + ctx.queue_record(&mut buffer, record.clone()); + assert_eq!(buffer.len(), 1); + assert_eq!(buffer.front(), Some(&record)); + + ctx.queue_record(&mut buffer, record.clone()); + assert_eq!(buffer.len(), 2); + + ctx.queue_record(&mut buffer, record.clone()); + assert_eq!(buffer.len(), 3); + + ctx.queue_record(&mut buffer, record.clone()); + assert_eq!(buffer.len(), 3, "Should not exceed 3 records"); + } + + #[rstest] + fn test_ctx_before_duration() { + let ctx = CtxBefore::new(Duration(5000)); + let mut buffer = VecDeque::new(); + let record1: TestFilterRecord = RecordMeta::builder().timestamp(1000).build().into(); + let record2: TestFilterRecord = RecordMeta::builder().timestamp(6000).build().into(); + let record3: TestFilterRecord = RecordMeta::builder().timestamp(6001).build().into(); + + ctx.queue_record(&mut buffer, record1.clone()); + assert_eq!(buffer.len(), 1); + assert_eq!(buffer.front(), Some(&record1)); + + ctx.queue_record(&mut buffer, record2.clone()); + assert_eq!(buffer.len(), 2); + assert_eq!(buffer.front(), Some(&record1)); + + ctx.queue_record(&mut buffer, record3.clone()); + assert_eq!( + buffer.len(), + 2, + "Should remove the first record after 5000ms" + ); + assert_eq!( + buffer.front(), + Some(&record2), + "Should keep the second record" + ); + } +} diff --git a/reductstore/src/storage/query/historical.rs b/reductstore/src/storage/query/historical.rs index 8430a5ef1..575055eae 100644 --- a/reductstore/src/storage/query/historical.rs +++ b/reductstore/src/storage/query/historical.rs @@ -1,19 +1,38 @@ -// Copyright 2023-2024 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 -use std::collections::VecDeque; -use std::sync::{Arc, RwLock}; - use crate::storage::block_manager::{BlockManager, BlockRef}; use crate::storage::entry::RecordReader; use crate::storage::proto::{record::State as RecordState, ts_to_us, Record}; use crate::storage::query::base::{Query, QueryOptions}; use crate::storage::query::condition::Parser; use crate::storage::query::filters::{ - EachNFilter, EachSecondFilter, ExcludeLabelFilter, IncludeLabelFilter, RecordFilter, - RecordStateFilter, TimeRangeFilter, WhenFilter, + apply_filters_recursively, EachNFilter, EachSecondFilter, ExcludeLabelFilter, FilterRecord, + IncludeLabelFilter, RecordFilter, RecordStateFilter, TimeRangeFilter, WhenFilter, }; -use reduct_base::error::{ErrorCode, ReductError}; +use reduct_base::error::ReductError; +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, RwLock}; + +impl FilterRecord for Record { + fn state(&self) -> i32 { + self.state + } + + fn timestamp(&self) -> u64 { + self.timestamp.as_ref().map_or(0, |ts| ts_to_us(ts)) + } + + fn labels(&self) -> HashMap<&String, &String> { + HashMap::from_iter(self.labels.iter().map(|label| (&label.name, &label.value))) + } + + fn computed_labels(&self) -> HashMap<&String, &String> { + HashMap::new() + } +} + +type Filter = Box + Send + Sync>; pub struct HistoricalQuery { /// The start time of the query. @@ -25,11 +44,9 @@ pub struct HistoricalQuery { /// The current block that is being read. Cached to avoid loading the same block multiple times. current_block: Option, /// Filters - filters: Vec>, + filters: Vec, /// Request only metadata without the content. only_metadata: bool, - /// Strict mode - strict: bool, /// Interrupted query is_interrupted: bool, } @@ -40,7 +57,7 @@ impl HistoricalQuery { stop_time: u64, options: QueryOptions, ) -> Result { - let mut filters: Vec> = vec![ + let mut filters: Vec = vec![ Box::new(TimeRangeFilter::new(start_time, stop_time)), Box::new(RecordStateFilter::new(RecordState::Finished)), ]; @@ -63,8 +80,12 @@ impl HistoricalQuery { if let Some(when) = options.when { let parser = Parser::new(); - let condition = parser.parse(&when)?; - filters.push(Box::new(WhenFilter::new(condition))); + let (condition, directives) = parser.parse(when)?; + filters.push(Box::new(WhenFilter::try_new( + condition, + directives, + options.strict, + )?)); } Ok(HistoricalQuery { @@ -74,7 +95,6 @@ impl HistoricalQuery { current_block: None, filters, only_metadata: options.only_metadata, - strict: options.strict, is_interrupted: false, }) } @@ -114,10 +134,9 @@ impl Query for HistoricalQuery { let block_ref = bm.load_block(block_id)?; self.current_block = Some(block_ref); - let (mut found_records, continue_query) = - self.filter_records_from_current_block()?; - self.is_interrupted = !continue_query; + let mut found_records = self.filter_records_from_current_block()?; found_records.sort_by_key(|rec| ts_to_us(rec.timestamp.as_ref().unwrap())); + self.records_from_current_block.extend(found_records); if !self.records_from_current_block.is_empty() { break; @@ -146,43 +165,23 @@ impl Query for HistoricalQuery { } impl HistoricalQuery { - fn filter_records_from_current_block(&mut self) -> Result<(Vec, bool), ReductError> { + fn filter_records_from_current_block(&mut self) -> Result, ReductError> { let block = self.current_block.as_ref().unwrap().read()?; let mut filtered_records = Vec::new(); for record in block.record_index().values() { - let mut include_record = true; - for filter in self.filters.iter_mut() { - match filter.filter(&record.clone().into()) { - Ok(false) => { - include_record = false; - break; - } - Ok(true) => {} - - Err(err) => { - // if the filter is interrupted, we return what we have - // and notify the caller to stop the query - if err.status == ErrorCode::Interrupt { - return Ok((filtered_records, false)); - } - - if self.strict { - // in strict mode, we return an error if a filter fails - return Err(err); - } - - // in non-strict mode, we ignore the record with the failed filter - include_record = false; - break; - } + match apply_filters_recursively(self.filters.as_mut_slice(), vec![record.clone()], 0)? { + Some(records) => { + filtered_records.extend(records); + } + None => { + // If the filtering is interrupted, we stop processing further records. + self.is_interrupted = true; + break; } - } - if include_record { - filtered_records.push(record.clone()); } } - Ok((filtered_records, true)) + Ok(filtered_records) } } From cd1b909b200c0bdc550f6fefe2975218c1aec2cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:02:47 +0200 Subject: [PATCH 47/93] Bump reqwest from 0.12.20 to 0.12.22 (#868) Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.12.20 to 0.12.22. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.12.20...v0.12.22) --- updated-dependencies: - dependency-name: reqwest dependency-version: 0.12.22 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 668bf03bd..8d9b61e7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2112,9 +2112,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.20" +version = "0.12.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" +checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" dependencies = [ "base64", "bytes", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index ce5c8721e..889c37559 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -67,7 +67,7 @@ prost = "0.13.1" [build-dependencies] prost-build = "0.13.1" -reqwest = { version = "0.12.20", default-features = false, features = ["rustls-tls", "blocking", "json"] } +reqwest = { version = "0.12.22", default-features = false, features = ["rustls-tls", "blocking", "json"] } chrono = "0.4.41" serde_json = "1.0.140" @@ -76,7 +76,7 @@ mockall = "0.13.1" rstest = "0.25.0" serial_test = "3.2.0" test-log = "0.2.17" -reqwest = { version = "0.12.20", default-features = false, features = ["rustls-tls", "blocking"] } +reqwest = { version = "0.12.22", default-features = false, features = ["rustls-tls", "blocking"] } assert_matches = "1.5" [package.metadata.docs.rs] From e345aa0e9c7be0612911af979000af07f99f6d7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:03:51 +0200 Subject: [PATCH 48/93] Bump tokio from 1.45.1 to 1.46.1 (#870) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.45.1 to 1.46.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.45.1...tokio-1.46.1) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.46.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 17 +++++++++++++++-- reduct_base/Cargo.toml | 2 +- reductstore/Cargo.toml | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d9b61e7c..3e9552037 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1261,6 +1261,17 @@ dependencies = [ "rustversion", ] +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2682,17 +2693,19 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "windows-sys 0.52.0", diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 2ce8bbd75..2e8ca0e7c 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -34,7 +34,7 @@ url = "2.5.4" http = "1.2.0" bytes = "1.10.0" async-trait = { version = "0.1.87" , optional = true } -tokio = { version = "1.45.1", optional = true, features = ["default", "rt", "time"] } +tokio = { version = "1.46.1", optional = true, features = ["default", "rt", "time"] } log = "0.4.0" thread-id = "5.0.0" futures = "0.3.31" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 889c37559..d7d5aa558 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -43,7 +43,7 @@ regex = "1.11.1" bytes = "1.10.1" axum = { version = "0.8.4", features = ["default", "macros"] } axum-extra = { version = "0.10.0", features = ["default", "typed-header"] } -tokio = { version = "1.45.1", features = ["full"] } +tokio = { version = "1.46.1", features = ["full"] } hyper = { version = "1.6.0", features = ["full"] } futures-util = "0.3.31" axum-server = { version = "0.7.1", features = ["tls-rustls"] } From d2619f6e236c31ff9ad4a6d3b4f8ff3cba49e785 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 16:06:02 +0200 Subject: [PATCH 49/93] Bump test-log from 0.2.17 to 0.2.18 (#869) Bumps [test-log](https://github.com/d-e-s-o/test-log) from 0.2.17 to 0.2.18. - [Release notes](https://github.com/d-e-s-o/test-log/releases) - [Changelog](https://github.com/d-e-s-o/test-log/blob/main/CHANGELOG.md) - [Commits](https://github.com/d-e-s-o/test-log/compare/v0.2.17...v0.2.18) --- updated-dependencies: - dependency-name: test-log dependency-version: 0.2.18 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- reductstore/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e9552037..0b1d3cf3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2576,9 +2576,9 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "test-log" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7f46083d221181166e5b6f6b1e5f1d499f3a76888826e6cb1d057554157cd0f" +checksum = "1e33b98a582ea0be1168eba097538ee8dd4bbe0f2b01b22ac92ea30054e5be7b" dependencies = [ "env_logger", "test-log-macros", @@ -2587,9 +2587,9 @@ dependencies = [ [[package]] name = "test-log-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "888d0c3c6db53c0fdab160d2ed5e12ba745383d3e85813f2ea0f2b1475ab553f" +checksum = "451b374529930d7601b1eef8d32bc79ae870b6079b069401709c2a8bf9e75f36" dependencies = [ "proc-macro2", "quote", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index d7d5aa558..1401af7eb 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -75,7 +75,7 @@ serde_json = "1.0.140" mockall = "0.13.1" rstest = "0.25.0" serial_test = "3.2.0" -test-log = "0.2.17" +test-log = "0.2.18" reqwest = { version = "0.12.22", default-features = false, features = ["rustls-tls", "blocking"] } assert_matches = "1.5" From 2e0858ee6642f9b976c8417a9623df9fb2dc4429 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 22:28:29 +0200 Subject: [PATCH 50/93] Bump rustls from 0.23.28 to 0.23.29 (#871) Bumps [rustls](https://github.com/rustls/rustls) from 0.23.28 to 0.23.29. - [Release notes](https://github.com/rustls/rustls/releases) - [Changelog](https://github.com/rustls/rustls/blob/main/CHANGELOG.md) - [Commits](https://github.com/rustls/rustls/compare/v/0.23.28...v/0.23.29) --- updated-dependencies: - dependency-name: rustls dependency-version: 0.23.29 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- reductstore/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0b1d3cf3a..e9177fadb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2262,9 +2262,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1" dependencies = [ "aws-lc-rs", "log", @@ -2297,9 +2297,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "aws-lc-rs", "ring", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 1401af7eb..8924607d1 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -58,7 +58,7 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus async-stream = "0.3.6" tower-http = { version = "0.6.6", features = ["cors"] } crc64fast = "1.1.0" -rustls = "0.23.27" +rustls = "0.23.29" byteorder = "1.5.0" crossbeam-channel = "0.5.15" dlopen2 = "0.8.0" From 08a14e936bfcf63f6faebbf4e7bf670246c3b1e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 22:28:42 +0200 Subject: [PATCH 51/93] Bump zip from 4.2.0 to 4.3.0 (#872) Bumps [zip](https://github.com/zip-rs/zip2) from 4.2.0 to 4.3.0. - [Release notes](https://github.com/zip-rs/zip2/releases) - [Changelog](https://github.com/zip-rs/zip2/blob/master/CHANGELOG.md) - [Commits](https://github.com/zip-rs/zip2/compare/v4.2.0...v4.3.0) --- updated-dependencies: - dependency-name: zip dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 33 ++++++++++++++++++--------------- reductstore/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9177fadb..76c7f9701 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,21 +393,11 @@ checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba" [[package]] name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" dependencies = [ - "cc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] @@ -1365,6 +1355,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "libbz2-rs-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775bf80d5878ab7c2b1080b5351a48b2f737d9f6f8b383574eebcc22be0dfccb" + [[package]] name = "libc" version = "0.2.174" @@ -1718,6 +1714,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -3502,9 +3504,9 @@ dependencies = [ [[package]] name = "zip" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ab361742de920c5535880f89bbd611ee62002bf11341d16a5f057bb8ba6899" +checksum = "9aed4ac33e8eb078c89e6cbb1d5c4c7703ec6d299fc3e7c3695af8f8b423468b" dependencies = [ "aes", "arbitrary", @@ -3519,6 +3521,7 @@ dependencies = [ "liblzma", "memchr", "pbkdf2", + "ppmd-rust", "sha1", "time", "zeroize", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 8924607d1..63c7a63cf 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -32,7 +32,7 @@ reduct-base = { path = "../reduct_base", version = "1.15.0", features = ["ext"] reduct-macros = { path = "../reduct_macros", version = "1.15.0" } chrono = { version = "0.4.41", features = ["serde"] } -zip = "4.1.0" +zip = "4.3.0" tempfile = "3.20.0" hex = "0.4.3" prost-wkt-types = "0.6.1" From f08897f404dbf055717d9f629f18d5ff773fde3f Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Fri, 18 Jul 2025 23:33:59 +0200 Subject: [PATCH 52/93] Remove deprecated `x-reduct-last` flag (#876) * remove deprecated x-reduct-last flag * clean code * update CHANGELOG --- CHANGELOG.md | 4 ++ integration_tests/api/entry_api/query_test.py | 2 - .../api/entry_api/read_write_record_test.py | 2 - reduct_base/src/io.rs | 60 +++++-------------- reductstore/src/api/entry/read_query.rs | 4 +- reductstore/src/api/entry/read_query_post.rs | 4 +- reductstore/src/api/entry/read_single.rs | 3 - reductstore/src/ext/ext_repository.rs | 2 +- reductstore/src/replication/remote_bucket.rs | 2 +- .../remote_bucket/client_wrapper.rs | 4 +- .../remote_bucket/states/bucket_available.rs | 2 - reductstore/src/storage/block_manager.rs | 10 +--- .../src/storage/entry/io/record_reader.rs | 21 +++---- reductstore/src/storage/entry/read_record.rs | 2 +- reductstore/src/storage/query/historical.rs | 3 +- reductstore/src/storage/query/limited.rs | 11 +--- 16 files changed, 40 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d186be28f..c6843472c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix hanging query request if no extension registered, [PR-830](https://github.com/reductstore/reductstore/pull/830) - Fix `$limit` operator in extension context, [PR-848](https://github.com/reductstore/reductstore/pull/848) +### Removed + +- `x-reduct-last` header from query response, [PR-876](https://github.com/reductstore/reductstore/pull/876) + ## [1.15.6] - 2025-06-25 ### Fixed diff --git a/integration_tests/api/entry_api/query_test.py b/integration_tests/api/entry_api/query_test.py index c548b6eef..d0cfe00d5 100644 --- a/integration_tests/api/entry_api/query_test.py +++ b/integration_tests/api/entry_api/query_test.py @@ -63,14 +63,12 @@ def test_query_entry_next(base_url, session, bucket): assert resp.status_code == 200 assert resp.content == b"some_data" assert resp.headers["x-reduct-time"] == "1000" - assert resp.headers["x-reduct-last"] == "0" resp = session.get(f"{base_url}/b/{bucket}/entry?q={query_id}") assert resp.status_code == 200 assert resp.content == b"some_data" assert resp.headers["x-reduct-time"] == "1100" - assert resp.headers["x-reduct-last"] == "0" resp = session.get(f"{base_url}/b/{bucket}/entry?q={query_id}") assert resp.status_code == 204 diff --git a/integration_tests/api/entry_api/read_write_record_test.py b/integration_tests/api/entry_api/read_write_record_test.py index 22f5ee4f1..cab4f3b0c 100644 --- a/integration_tests/api/entry_api/read_write_record_test.py +++ b/integration_tests/api/entry_api/read_write_record_test.py @@ -50,7 +50,6 @@ def test_read_write_entries_big_blob_ok(base_url, session, bucket): assert resp.headers["content-type"] == "application/octet-stream" assert resp.headers["x-reduct-time"] == str(ts) - assert resp.headers["x-reduct-last"] == "1" @requires_env("API_TOKEN") @@ -111,7 +110,6 @@ def test_latest_record(base_url, session, bucket): assert resp.status_code == 200 assert resp.content == b"some_data2" assert resp.headers["x-reduct-time"] == "1010" - assert resp.headers["x-reduct-last"] == "1" def test_read_write_big_blob(base_url, session, bucket): diff --git a/reduct_base/src/io.rs b/reduct_base/src/io.rs index 45fc45737..6219173d6 100644 --- a/reduct_base/src/io.rs +++ b/reduct_base/src/io.rs @@ -17,7 +17,6 @@ pub struct RecordMeta { content_type: String, content_length: u64, computed_labels: Labels, - last: bool, } pub struct BuilderRecordMeta { @@ -27,7 +26,6 @@ pub struct BuilderRecordMeta { content_type: String, content_length: u64, computed_labels: Labels, - last: bool, } impl BuilderRecordMeta { @@ -67,11 +65,6 @@ impl BuilderRecordMeta { self } - pub fn last(mut self, last: bool) -> Self { - self.last = last; - self - } - /// Builds a `RecordMeta` instance from the builder. pub fn build(self) -> RecordMeta { RecordMeta { @@ -81,7 +74,6 @@ impl BuilderRecordMeta { content_type: self.content_type, content_length: self.content_length, computed_labels: self.computed_labels, - last: self.last, } } } @@ -96,7 +88,6 @@ impl RecordMeta { content_type: "application/octet-stream".to_string(), content_length: 0, computed_labels: Labels::new(), - last: false, } } @@ -109,7 +100,6 @@ impl RecordMeta { content_type: meta.content_type, content_length: meta.content_length, computed_labels: meta.computed_labels, - last: meta.last, } } @@ -128,11 +118,6 @@ impl RecordMeta { self.state } - /// Returns true if this is the last record in the stream. - pub fn last(&self) -> bool { - self.last - } - /// Returns computed labels associated with the record. /// /// Computed labels are labels that are added by query processing and are not part of the original record. @@ -156,10 +141,6 @@ impl RecordMeta { pub fn content_type(&self) -> &str { &self.content_type } - - pub fn set_last(&mut self, last: bool) { - self.last = last; - } } /// Represents a record in the storage engine that can be read as a stream of bytes. @@ -220,7 +201,7 @@ pub trait WriteRecord { pub(crate) mod tests { use super::*; - use rstest::rstest; + use rstest::{fixture, rstest}; use tokio::task::spawn_blocking; @@ -255,36 +236,15 @@ pub(crate) mod tests { use super::*; #[rstest] - fn test_builder() { - let meta = RecordMeta::builder() - .timestamp(1234567890) - .state(1) - .labels(Labels::new()) - .content_type("application/json".to_string()) - .content_length(1024) - .computed_labels(Labels::new()) - .last(true) - .build(); - + fn test_builder(meta: RecordMeta) { assert_eq!(meta.timestamp(), 1234567890); assert_eq!(meta.state(), 1); assert_eq!(meta.content_type(), "application/json"); assert_eq!(meta.content_length(), 1024); - assert_eq!(meta.last(), true); } #[rstest] - fn test_builder_from() { - let meta = RecordMeta::builder() - .timestamp(1234567890) - .state(1) - .labels(Labels::new()) - .content_type("application/json".to_string()) - .content_length(1024) - .computed_labels(Labels::new()) - .last(true) - .build(); - + fn test_builder_from(meta: RecordMeta) { let builder = RecordMeta::builder_from(meta.clone()); let new_meta = builder.build(); @@ -292,7 +252,18 @@ pub(crate) mod tests { assert_eq!(new_meta.state(), 1); assert_eq!(new_meta.content_type(), "application/json"); assert_eq!(new_meta.content_length(), 1024); - assert_eq!(new_meta.last(), true); + } + + #[fixture] + fn meta() -> RecordMeta { + RecordMeta::builder() + .timestamp(1234567890) + .state(1) + .labels(Labels::new()) + .content_type("application/json".to_string()) + .content_length(1024) + .computed_labels(Labels::new()) + .build() } } @@ -310,7 +281,6 @@ pub(crate) mod tests { .content_type("application/octet-stream".to_string()) .content_length(0) .computed_labels(Labels::new()) - .last(false) .build(), } } diff --git a/reductstore/src/api/entry/read_query.rs b/reductstore/src/api/entry/read_query.rs index 08086a640..4d569b747 100644 --- a/reductstore/src/api/entry/read_query.rs +++ b/reductstore/src/api/entry/read_query.rs @@ -44,7 +44,7 @@ mod tests { use super::*; use crate::api::tests::{components, headers, path_to_entry_1}; use reduct_base::error::ErrorCode; - use reduct_base::io::ReadRecord; + use rstest::*; #[rstest] @@ -84,7 +84,7 @@ mod tests { .upgrade() .unwrap(); let mut rx = rx.write().await; - assert!(rx.recv().await.unwrap().unwrap().meta().last()); + assert!(rx.recv().await.unwrap().is_ok()); assert_eq!( rx.recv().await.unwrap().err().unwrap().status, diff --git a/reductstore/src/api/entry/read_query_post.rs b/reductstore/src/api/entry/read_query_post.rs index 67c3365f0..c0d8a4d57 100644 --- a/reductstore/src/api/entry/read_query_post.rs +++ b/reductstore/src/api/entry/read_query_post.rs @@ -50,7 +50,7 @@ mod tests { use crate::core::weak::Weak; use crate::storage::query::QueryRx; use reduct_base::error::{ErrorCode, ReductError}; - use reduct_base::io::ReadRecord; + use reduct_base::msg::entry_api::QueryType; use rstest::*; use serde_json::json; @@ -76,7 +76,7 @@ mod tests { .upgrade() .unwrap(); let mut rx = rx.write().await; - assert!(rx.recv().await.unwrap().unwrap().meta().last()); + assert!(rx.recv().await.unwrap().is_ok()); assert_eq!( rx.recv().await.unwrap().err().unwrap().status, ErrorCode::NoContent diff --git a/reductstore/src/api/entry/read_single.rs b/reductstore/src/api/entry/read_single.rs index e775f03ae..38bb0324e 100644 --- a/reductstore/src/api/entry/read_single.rs +++ b/reductstore/src/api/entry/read_single.rs @@ -24,7 +24,6 @@ use hyper::http::HeaderValue; use reduct_base::bad_request; use reduct_base::io::ReadRecord; use std::collections::HashMap; -use std::i64; use std::pin::{pin, Pin}; use std::sync::Arc; use std::task::{Context, Poll}; @@ -86,7 +85,6 @@ async fn fetch_and_response_single_record( ); headers.insert("content-length", HeaderValue::from(meta.content_length())); headers.insert("x-reduct-time", HeaderValue::from(meta.timestamp())); - headers.insert("x-reduct-last", HeaderValue::from(i64::from(meta.last()))); headers }; @@ -387,7 +385,6 @@ mod tests { labels: vec![], content_type: "".to_string(), }, - false, ), empty_body: false, }; diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index ada1c73dc..da81a49ee 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -874,7 +874,7 @@ pub(super) mod tests { }), ..Default::default() }; - RecordReader::form_record(record, false) + RecordReader::form_record(record) } #[fixture] diff --git a/reductstore/src/replication/remote_bucket.rs b/reductstore/src/replication/remote_bucket.rs index 5632a63bb..d852748f7 100644 --- a/reductstore/src/replication/remote_bucket.rs +++ b/reductstore/src/replication/remote_bucket.rs @@ -203,7 +203,7 @@ pub(super) mod tests { let (_tx, rx) = tokio::sync::mpsc::channel(1); let mut rec = Record::default(); rec.timestamp = Some(Timestamp::default()); - let record = RecordReader::form_record_with_rx(rx, rec, false); + let record = RecordReader::form_record_with_rx(rx, rec); remote_bucket.write_batch("test", vec![(record, Transaction::WriteRecord(0))]) } } diff --git a/reductstore/src/replication/remote_bucket/client_wrapper.rs b/reductstore/src/replication/remote_bucket/client_wrapper.rs index 364967ec0..8ae8d5d7e 100644 --- a/reductstore/src/replication/remote_bucket/client_wrapper.rs +++ b/reductstore/src/replication/remote_bucket/client_wrapper.rs @@ -500,8 +500,8 @@ mod tests { ( vec![ - RecordReader::form_record_with_rx(rx1, rec1, false), - RecordReader::form_record_with_rx(rx2, rec2, false), + RecordReader::form_record_with_rx(rx1, rec1), + RecordReader::form_record_with_rx(rx2, rec2), ], vec![tx1, tx2], ) diff --git a/reductstore/src/replication/remote_bucket/states/bucket_available.rs b/reductstore/src/replication/remote_bucket/states/bucket_available.rs index 28b214a4a..bdc5d0875 100644 --- a/reductstore/src/replication/remote_bucket/states/bucket_available.rs +++ b/reductstore/src/replication/remote_bucket/states/bucket_available.rs @@ -394,7 +394,6 @@ mod tests { content_type: "text/plain".to_string(), state: 0, }, - false, ), Transaction::WriteRecord(0), ) @@ -414,7 +413,6 @@ mod tests { content_type: "text/plain".to_string(), state: 0, }, - false, ), Transaction::UpdateRecord(0), ) diff --git a/reductstore/src/storage/block_manager.rs b/reductstore/src/storage/block_manager.rs index d4cde46d2..44132926a 100644 --- a/reductstore/src/storage/block_manager.rs +++ b/reductstore/src/storage/block_manager.rs @@ -955,13 +955,9 @@ mod tests { assert_eq!(record.state, 1); // read content - let mut reader = RecordReader::try_new( - Arc::clone(&block_manager), - block_ref.clone(), - record_time, - false, - ) - .unwrap(); + let mut reader = + RecordReader::try_new(Arc::clone(&block_manager), block_ref.clone(), record_time) + .unwrap(); let mut received = BytesMut::new(); while let Some(Ok(chunk)) = reader.blocking_read() { diff --git a/reductstore/src/storage/entry/io/record_reader.rs b/reductstore/src/storage/entry/io/record_reader.rs index 9a96b3447..e43aff768 100644 --- a/reductstore/src/storage/entry/io/record_reader.rs +++ b/reductstore/src/storage/entry/io/record_reader.rs @@ -56,7 +56,6 @@ impl RecordReader { block_manager: Arc>, block_ref: BlockRef, record_timestamp: u64, - last: bool, ) -> Result { let (record, ctx) = { let bm = block_manager.write()?; @@ -107,7 +106,7 @@ impl RecordReader { }); }; - Ok(Self::form_record_with_rx(rx, record, last)) + Ok(Self::form_record_with_rx(rx, record)) } /// Create a new record reader for a record with no content. @@ -117,27 +116,21 @@ impl RecordReader { /// # Arguments /// /// * `record` - The record to read - /// * `last` - Whether this is the last record in the entry /// /// # Returns /// /// * `RecordReader` - The record reader to read the record content in chunks - pub fn form_record(record: Record, last: bool) -> Self { - let mut meta: RecordMeta = record.into(); - meta.set_last(last); + pub fn form_record(record: Record) -> Self { + let meta: RecordMeta = record.into(); RecordReader { rx: None, meta } } - pub fn form_record_with_rx(rx: RecordRx, record: Record, last: bool) -> Self { - let mut me = Self::form_record(record, last); + pub fn form_record_with_rx(rx: RecordRx, record: Record) -> Self { + let mut me = Self::form_record(record); me.rx = Some(rx); me } - pub fn set_last(&mut self, last: bool) { - self.meta.set_last(last); - } - fn read(tx: Sender>, ctx: ReadContext) { let mut read_bytes = 0; let path = format!( @@ -400,7 +393,7 @@ mod tests { #[rstest] fn test_state(mut record: Record) { record.state = 1; - let reader = RecordReader::form_record(record, false); + let reader = RecordReader::form_record(record); assert_eq!(reader.meta().state(), 1); } @@ -408,7 +401,7 @@ mod tests { #[tokio::test] async fn test_read_timeout(record: Record) { let (_tx, rx) = channel(CHANNEL_BUFFER_SIZE); - let mut reader = RecordReader::form_record_with_rx(rx, record, false); + let mut reader = RecordReader::form_record_with_rx(rx, record); let result = reader.read_timeout(Duration::from_millis(100)).await; assert_eq!( diff --git a/reductstore/src/storage/entry/read_record.rs b/reductstore/src/storage/entry/read_record.rs index ce26fa038..eeda4ed0d 100644 --- a/reductstore/src/storage/entry/read_record.rs +++ b/reductstore/src/storage/entry/read_record.rs @@ -49,7 +49,7 @@ impl Entry { )); } - RecordReader::try_new(block_manager, block_ref, time, true) + RecordReader::try_new(block_manager, block_ref, time) }) } } diff --git a/reductstore/src/storage/query/historical.rs b/reductstore/src/storage/query/historical.rs index 575055eae..b50a92195 100644 --- a/reductstore/src/storage/query/historical.rs +++ b/reductstore/src/storage/query/historical.rs @@ -152,13 +152,12 @@ impl Query for HistoricalQuery { let block = self.current_block.as_ref().unwrap(); if self.only_metadata { - Ok(RecordReader::form_record(record.clone(), false)) + Ok(RecordReader::form_record(record.clone())) } else { RecordReader::try_new( Arc::clone(&block_manager), block.clone(), ts_to_us(&record.timestamp.unwrap()), - false, ) } } diff --git a/reductstore/src/storage/query/limited.rs b/reductstore/src/storage/query/limited.rs index cae9349b5..8dd538c25 100644 --- a/reductstore/src/storage/query/limited.rs +++ b/reductstore/src/storage/query/limited.rs @@ -35,15 +35,7 @@ impl Query for LimitedQuery { } self.limit_count -= 1; - let reader = self.query.next(block_manager); - if self.limit_count == 0 { - reader.map(|mut r| { - r.set_last(true); - r - }) - } else { - reader - } + self.query.next(block_manager) } } @@ -69,7 +61,6 @@ mod tests { let reader = query.next(block_manager.clone()).unwrap(); assert_eq!(reader.meta().timestamp(), 0); - assert!(reader.meta().last()); assert_eq!( query.next(block_manager).err(), From 66ea248dd0d6c8ff552e1d7c421811f37d45f5a7 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Sun, 20 Jul 2025 22:52:46 +0200 Subject: [PATCH 53/93] Support for wild card in write/read tokens (#877) * support for wild card in write/read tokens * test single asterisk * update CHANGELOG * refactoring * update api tests * improve CHANGELOG --- CHANGELOG.md | 1 + integration_tests/api/bucket_api_test.py | 33 ++-- integration_tests/api/conftest.py | 22 ++- integration_tests/api/entry_api/entry_test.py | 16 +- integration_tests/api/entry_api/query_test.py | 6 +- .../api/entry_api/read_write_record_test.py | 24 +-- .../api/entry_api/remove_record_test.py | 20 +-- integration_tests/api/server_api_test.py | 4 +- integration_tests/api/token_api_test.py | 30 +++- reduct_base/src/error.rs | 10 ++ reductstore/src/api/bucket/get.rs | 2 +- reductstore/src/api/entry/read_batched.rs | 2 +- reductstore/src/api/entry/read_query.rs | 2 +- reductstore/src/api/entry/read_query_post.rs | 2 +- reductstore/src/api/entry/read_single.rs | 2 +- reductstore/src/api/entry/remove_batched.rs | 2 +- reductstore/src/api/entry/remove_entry.rs | 2 +- reductstore/src/api/entry/remove_query.rs | 2 +- .../src/api/entry/remove_query_post.rs | 2 +- reductstore/src/api/entry/remove_single.rs | 9 +- reductstore/src/api/entry/rename_entry.rs | 2 +- reductstore/src/api/entry/update_batched.rs | 2 +- reductstore/src/api/entry/update_single.rs | 9 +- reductstore/src/api/entry/write_batched.rs | 4 +- reductstore/src/api/entry/write_single.rs | 9 +- reductstore/src/api/server/list.rs | 2 +- reductstore/src/auth/policy.rs | 141 ++++++++++-------- 27 files changed, 199 insertions(+), 163 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6843472c..3251da30d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Filter buckets by read permission in server information, [PR-849](https://github.com/reductstore/reductstore/pull/849) - Support for duration literals, [PR-864](https://github.com/reductstore/reductstore/pull/864) - Support for `#ctx_before` and `#ctx_after` directives, [PR-866](https://github.com/reductstore/reductstore/pull/866) +- Support for wildcards in write/read token permissions, [PR-877](https://github.com/reductstore/reductstore/pull/877) ### Changed diff --git a/integration_tests/api/bucket_api_test.py b/integration_tests/api/bucket_api_test.py index f8b36dc34..6683e1552 100644 --- a/integration_tests/api/bucket_api_test.py +++ b/integration_tests/api/bucket_api_test.py @@ -46,7 +46,8 @@ def test__create_bucket_with_full_access_token( assert resp.status_code == 401 resp = session.post( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_without_permissions) + f"{base_url}/b/{bucket_name}", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 @@ -94,17 +95,18 @@ def test__get_bucket_with_authenticated_token( assert resp.status_code == 401 resp = session.get( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_without_permissions) + f"{base_url}/b/{bucket_name}", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.get( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket) + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket.value) ) assert resp.status_code == 200 resp = session.get( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket) + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket.value) ) assert resp.status_code == 403 @@ -201,17 +203,18 @@ def test__update_bucket_with_full_access_token( assert resp.status_code == 401 resp = session.put( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_without_permissions) + f"{base_url}/b/{bucket_name}", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.put( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket) + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket.value) ) assert resp.status_code == 403 resp = session.put( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket) + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket.value) ) assert resp.status_code == 403 @@ -242,17 +245,18 @@ def test__remove_bucket_with_full_access_token( assert resp.status_code == 401 resp = session.delete( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_without_permissions) + f"{base_url}/b/{bucket_name}", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.delete( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket) + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_read_bucket.value) ) assert resp.status_code == 403 resp = session.delete( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket) + f"{base_url}/b/{bucket_name}", headers=auth_headers(token_write_bucket.value) ) assert resp.status_code == 403 @@ -283,7 +287,8 @@ def test__head_bucket_with_full_access_token( assert resp.status_code == 401 resp = session.head( - f"{base_url}/b/{bucket_name}", headers=auth_headers(token_without_permissions) + f"{base_url}/b/{bucket_name}", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 200 @@ -327,20 +332,20 @@ def test__rename_bucket_with_full_access_token( resp = session.put( f"{base_url}/b/{bucket_name}/rename", json={"new_name": new_bucket_name}, - headers=auth_headers(token_without_permissions), + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.put( f"{base_url}/b/{bucket_name}/rename", json={"new_name": new_bucket_name}, - headers=auth_headers(token_read_bucket), + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.put( f"{base_url}/b/{bucket_name}/rename", json={"new_name": new_bucket_name}, - headers=auth_headers(token_write_bucket), + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 403 diff --git a/integration_tests/api/conftest.py b/integration_tests/api/conftest.py index e2cf95461..d6e9f0447 100644 --- a/integration_tests/api/conftest.py +++ b/integration_tests/api/conftest.py @@ -2,6 +2,7 @@ import os import random import secrets +from collections import namedtuple from typing import Callable import pytest @@ -75,9 +76,12 @@ def _make_token_permissions(session, base_url, token_generator): permissions = { "full_access": False, } - resp = session.post(f"{base_url}/tokens/{token_generator()}", json=permissions) + name = token_generator() + resp = session.post(f"{base_url}/tokens/{name}", json=permissions) assert resp.status_code == 200 - return json.loads(resp.content)["value"] + return namedtuple("Token", ["name", "value"])( + name, json.loads(resp.content)["value"] + ) @pytest.fixture(name="token_read_bucket") @@ -87,9 +91,12 @@ def _make_token_read_bucket(session, base_url, bucket_name, token_generator): "full_access": False, "read": [bucket_name], } - resp = session.post(f"{base_url}/tokens/{token_generator()}", json=permissions) + name = token_generator() + resp = session.post(f"{base_url}/tokens/{name}", json=permissions) assert resp.status_code == 200 - return json.loads(resp.content)["value"] + return namedtuple("Token", ["name", "value"])( + name, json.loads(resp.content)["value"] + ) @pytest.fixture(name="token_write_bucket") @@ -100,6 +107,9 @@ def _make_token_write_bucket(session, base_url, bucket_name, token_generator): "full_access": False, "write": [bucket_name], } - resp = session.post(f"{base_url}/tokens/{token_generator()}", json=permissions) + name = token_generator() + resp = session.post(f"{base_url}/tokens/{name}", json=permissions) assert resp.status_code == 200 - return json.loads(resp.content)["value"] + return namedtuple("Token", ["name", "value"])( + name, json.loads(resp.content)["value"] + ) diff --git a/integration_tests/api/entry_api/entry_test.py b/integration_tests/api/entry_api/entry_test.py index d796ef09c..801249895 100644 --- a/integration_tests/api/entry_api/entry_test.py +++ b/integration_tests/api/entry_api/entry_test.py @@ -1,4 +1,4 @@ -from ..conftest import requires_env +from ..conftest import requires_env, auth_headers def test_remove_entry(base_url, session, bucket): @@ -29,19 +29,19 @@ def test_remove_with_bucket_write_permissions( resp = session.delete( f"{base_url}/b/{bucket}/entry", - headers={"Authorization": f"Bearer {token_without_permissions}"}, + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry", - headers={"Authorization": f"Bearer {token_read_bucket}"}, + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry", - headers={"Authorization": f"Bearer {token_write_bucket}"}, + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 @@ -74,27 +74,27 @@ def test_rename_with_bucket_write_permissions( token_read_bucket, token_write_bucket, ): - """Needs write permissions to rename entry""" + """Needs to write permissions to rename entry""" resp = session.post(f"{base_url}/b/{bucket}/entry?ts=1000", data="some_data1") assert resp.status_code == 200 resp = session.put( f"{base_url}/b/{bucket}/entry/rename", json={"new_name": "new_name"}, - headers={"Authorization": f"Bearer {token_without_permissions}"}, + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.put( f"{base_url}/b/{bucket}/entry/rename", json={"new_name": "new_name"}, - headers={"Authorization": f"Bearer {token_read_bucket}"}, + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.put( f"{base_url}/b/{bucket}/entry/rename", json={"new_name": "new_name"}, - headers={"Authorization": f"Bearer {token_write_bucket}"}, + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 diff --git a/integration_tests/api/entry_api/query_test.py b/integration_tests/api/entry_api/query_test.py index d0cfe00d5..29656f477 100644 --- a/integration_tests/api/entry_api/query_test.py +++ b/integration_tests/api/entry_api/query_test.py @@ -292,20 +292,20 @@ def test__query_with_read_token( resp = session.get( f"{base_url}/b/{bucket}/entry/q", - headers=auth_headers(token_without_permissions), + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.post( f"{base_url}/b/{bucket}/entry/q", - headers=auth_headers(token_read_bucket), + headers=auth_headers(token_read_bucket.value), json={"query_type": "QUERY"}, ) assert resp.status_code == 404 # no data resp = session.post( f"{base_url}/b/{bucket}/entry/q", - headers=auth_headers(token_write_bucket), + headers=auth_headers(token_write_bucket.value), json={"query_type": "QUERY"}, ) assert resp.status_code == 403 diff --git a/integration_tests/api/entry_api/read_write_record_test.py b/integration_tests/api/entry_api/read_write_record_test.py index cab4f3b0c..bbbe3a81f 100644 --- a/integration_tests/api/entry_api/read_write_record_test.py +++ b/integration_tests/api/entry_api/read_write_record_test.py @@ -67,17 +67,19 @@ def test__read_with_read_token( resp = session.get( f"{base_url}/b/{bucket}/entry?ts=1000", - headers=auth_headers(token_without_permissions), + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.get( - f"{base_url}/b/{bucket}/entry?ts=1000", headers=auth_headers(token_read_bucket) + f"{base_url}/b/{bucket}/entry?ts=1000", + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 404 # no data resp = session.get( - f"{base_url}/b/{bucket}/entry?ts=1000", headers=auth_headers(token_write_bucket) + f"{base_url}/b/{bucket}/entry?ts=1000", + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 403 @@ -150,7 +152,7 @@ def test__write_with_write_token( token_read_bucket, token_write_bucket, ): - """Needs write permissions to write""" + """Needs to write permissions to write""" resp = session.post( f"{base_url}/b/{bucket}/entry?ts=1000", headers=auth_headers("") ) @@ -158,17 +160,19 @@ def test__write_with_write_token( resp = session.post( f"{base_url}/b/{bucket}/entry?ts=1000", - headers=auth_headers(token_without_permissions), + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.post( - f"{base_url}/b/{bucket}/entry?ts=1000", headers=auth_headers(token_read_bucket) + f"{base_url}/b/{bucket}/entry?ts=1000", + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.post( - f"{base_url}/b/{bucket}/entry?ts=1000", headers=auth_headers(token_write_bucket) + f"{base_url}/b/{bucket}/entry?ts=1000", + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 @@ -178,7 +182,6 @@ def test__head_entry_with_full_access_token( base_url, session, bucket, - token_without_permissions, token_write_bucket, token_read_bucket, ): @@ -189,12 +192,13 @@ def test__head_entry_with_full_access_token( resp = session.post( f"{base_url}/b/{bucket}/{entry_name}?ts={ts}", data=dummy_data, - headers=auth_headers(token_write_bucket), + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 resp = session.head( - f"{base_url}/b/{bucket}/{entry_name}", headers=auth_headers(token_read_bucket) + f"{base_url}/b/{bucket}/{entry_name}", + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 200 assert len(resp.content) == 0 diff --git a/integration_tests/api/entry_api/remove_record_test.py b/integration_tests/api/entry_api/remove_record_test.py index 0963666ee..e7bc41d9d 100644 --- a/integration_tests/api/entry_api/remove_record_test.py +++ b/integration_tests/api/entry_api/remove_record_test.py @@ -1,5 +1,5 @@ import pytest -from ..conftest import requires_env +from ..conftest import requires_env, auth_headers @pytest.fixture(name="write_data") @@ -92,21 +92,21 @@ def test_remove_record_with_permission( resp = session.delete( f"{base_url}/b/{bucket}/entry?ts={ts1}", - headers={"Authorization": f"Bearer {token_without_permissions}"}, + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry?ts={ts1}", - headers={"Authorization": f"Bearer {token_read_bucket}"}, + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry?ts={ts1}", - headers={"Authorization": f"Bearer {token_write_bucket}"}, + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 @@ -126,21 +126,21 @@ def test_remove_records_in_batch_with_permission( resp = session.delete( f"{base_url}/b/{bucket}/entry/batch", - headers={"Authorization": f"Bearer {token_without_permissions}"}, + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry/batch", - headers={"Authorization": f"Bearer {token_read_bucket}"}, + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry/batch", - headers={"Authorization": f"Bearer {token_write_bucket}"}, + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 @@ -160,21 +160,21 @@ def test_remove_records_query_with_permission( resp = session.delete( f"{base_url}/b/{bucket}/entry/q?start={ts1}&stop={ts2 + 1}", - headers={"Authorization": f"Bearer {token_without_permissions}"}, + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry/q?start={ts1}&stop={ts2 + 1}", - headers={"Authorization": f"Bearer {token_read_bucket}"}, + headers=auth_headers(token_read_bucket.value), ) assert resp.status_code == 403 resp = session.delete( f"{base_url}/b/{bucket}/entry/q?start={ts1}&stop={ts2 + 1}", - headers={"Authorization": f"Bearer {token_write_bucket}"}, + headers=auth_headers(token_write_bucket.value), ) assert resp.status_code == 200 diff --git a/integration_tests/api/server_api_test.py b/integration_tests/api/server_api_test.py index b2c494f3c..b69db9d25 100644 --- a/integration_tests/api/server_api_test.py +++ b/integration_tests/api/server_api_test.py @@ -51,7 +51,7 @@ def test__authorized_info(base_url, session, token_without_permissions): assert resp.status_code == 401 resp = session.get( - f"{base_url}/info", headers=auth_headers(token_without_permissions) + f"{base_url}/info", headers=auth_headers(token_without_permissions.value) ) assert resp.status_code == 200 @@ -80,7 +80,7 @@ def test__authorized_list(base_url, session, token_without_permissions): assert resp.status_code == 401 resp = session.get( - f"{base_url}/list", headers=auth_headers(token_without_permissions) + f"{base_url}/list", headers=auth_headers(token_without_permissions.value) ) assert resp.status_code == 200 diff --git a/integration_tests/api/token_api_test.py b/integration_tests/api/token_api_test.py index 69ef8a1ce..c0b49a5fa 100644 --- a/integration_tests/api/token_api_test.py +++ b/integration_tests/api/token_api_test.py @@ -36,10 +36,13 @@ def test__creat_token_with_full_access( resp = session.post( f"{base_url}/tokens/{token_name}", json=permissions, - headers=auth_headers(token_without_permissions), + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 - assert resp.headers["x-reduct-error"] == "Token doesn't have full access" + assert ( + resp.headers["x-reduct-error"] + == f"Token '{token_without_permissions.name}' doesn't have full access" + ) @requires_env("API_TOKEN") @@ -63,10 +66,13 @@ def test__list_token_with_full_access(base_url, session, token_without_permissio assert resp.status_code == 401 resp = session.get( - f"{base_url}/tokens", headers=auth_headers(token_without_permissions) + f"{base_url}/tokens", headers=auth_headers(token_without_permissions.value) ) assert resp.status_code == 403 - assert resp.headers["x-reduct-error"] == "Token doesn't have full access" + assert ( + resp.headers["x-reduct-error"] + == f"Token '{token_without_permissions.name}' doesn't have full access" + ) @requires_env("API_TOKEN") @@ -104,10 +110,14 @@ def test__get_token_with_full_access(base_url, session, token_without_permission assert resp.status_code == 401 resp = session.get( - f"{base_url}/tokens/token-name", headers=auth_headers(token_without_permissions) + f"{base_url}/tokens/token-name", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 - assert resp.headers["x-reduct-error"] == "Token doesn't have full access" + assert ( + resp.headers["x-reduct-error"] + == f"Token '{token_without_permissions.name}' doesn't have full access" + ) @requires_env("API_TOKEN") @@ -133,7 +143,11 @@ def test__delete_token_with_full_access(base_url, session, token_without_permiss assert resp.status_code == 401 resp = session.delete( - f"{base_url}/tokens/token-name", headers=auth_headers(token_without_permissions) + f"{base_url}/tokens/token-name", + headers=auth_headers(token_without_permissions.value), ) assert resp.status_code == 403 - assert resp.headers["x-reduct-error"] == "Token doesn't have full access" + assert ( + resp.headers["x-reduct-error"] + == f"Token '{token_without_permissions.name}' doesn't have full access" + ) diff --git a/reduct_base/src/error.rs b/reduct_base/src/error.rs index ea6c09aae..1e4d30d3f 100644 --- a/reduct_base/src/error.rs +++ b/reduct_base/src/error.rs @@ -293,6 +293,16 @@ macro_rules! bad_request { }; } +#[macro_export] +macro_rules! forbidden { + ($msg:expr, $($arg:tt)*) => { + ReductError::forbidden(&format!($msg, $($arg)*)) + }; + ($msg:expr) => { + ReductError::forbidden($msg) + }; +} + #[macro_export] macro_rules! unprocessable_entity { ($msg:expr, $($arg:tt)*) => { diff --git a/reductstore/src/api/bucket/get.rs b/reductstore/src/api/bucket/get.rs index b0b29bae9..c20a548d3 100644 --- a/reductstore/src/api/bucket/get.rs +++ b/reductstore/src/api/bucket/get.rs @@ -21,7 +21,7 @@ pub(crate) async fn get_bucket( &components, &headers, ReadAccessPolicy { - bucket: bucket_name.clone(), + bucket: &bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index 7d87541d9..e9de749bd 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -46,7 +46,7 @@ pub(crate) async fn read_batched_records( &components, &headers, ReadAccessPolicy { - bucket: bucket_name.clone(), + bucket: &bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/read_query.rs b/reductstore/src/api/entry/read_query.rs index 4d569b747..8bb57c722 100644 --- a/reductstore/src/api/entry/read_query.rs +++ b/reductstore/src/api/entry/read_query.rs @@ -27,7 +27,7 @@ pub(crate) async fn read_query( &components, &headers, ReadAccessPolicy { - bucket: bucket_name.clone(), + bucket: &bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/read_query_post.rs b/reductstore/src/api/entry/read_query_post.rs index c0d8a4d57..9627b1c6d 100644 --- a/reductstore/src/api/entry/read_query_post.rs +++ b/reductstore/src/api/entry/read_query_post.rs @@ -26,7 +26,7 @@ pub(crate) async fn read_query_json( &components, &headers, ReadAccessPolicy { - bucket: bucket_name.clone(), + bucket: &bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/read_single.rs b/reductstore/src/api/entry/read_single.rs index 38bb0324e..9309e363c 100644 --- a/reductstore/src/api/entry/read_single.rs +++ b/reductstore/src/api/entry/read_single.rs @@ -43,7 +43,7 @@ pub(crate) async fn read_record( &components, &headers, ReadAccessPolicy { - bucket: bucket_name.clone(), + bucket: &bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/remove_batched.rs b/reductstore/src/api/entry/remove_batched.rs index 5bd40085e..d8ad85735 100644 --- a/reductstore/src/api/entry/remove_batched.rs +++ b/reductstore/src/api/entry/remove_batched.rs @@ -27,7 +27,7 @@ pub(crate) async fn remove_batched_records( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/remove_entry.rs b/reductstore/src/api/entry/remove_entry.rs index a2a1ca49e..3d13bd45e 100644 --- a/reductstore/src/api/entry/remove_entry.rs +++ b/reductstore/src/api/entry/remove_entry.rs @@ -24,7 +24,7 @@ pub(crate) async fn remove_entry( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/remove_query.rs b/reductstore/src/api/entry/remove_query.rs index b7ee36c34..79ec2a821 100644 --- a/reductstore/src/api/entry/remove_query.rs +++ b/reductstore/src/api/entry/remove_query.rs @@ -29,7 +29,7 @@ pub(crate) async fn remove_query( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/remove_query_post.rs b/reductstore/src/api/entry/remove_query_post.rs index 4679c233e..3e5496cd6 100644 --- a/reductstore/src/api/entry/remove_query_post.rs +++ b/reductstore/src/api/entry/remove_query_post.rs @@ -34,7 +34,7 @@ pub(crate) async fn remove_query_json( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/remove_single.rs b/reductstore/src/api/entry/remove_single.rs index ba2ce152f..def09893a 100644 --- a/reductstore/src/api/entry/remove_single.rs +++ b/reductstore/src/api/entry/remove_single.rs @@ -20,14 +20,7 @@ pub(crate) async fn remove_record( Query(params): Query>, ) -> Result<(), HttpError> { let bucket = path.get("bucket_name").unwrap(); - check_permissions( - &components, - &headers, - WriteAccessPolicy { - bucket: bucket.clone(), - }, - ) - .await?; + check_permissions(&components, &headers, WriteAccessPolicy { bucket }).await?; let ts = parse_timestamp_from_query(¶ms)?; let entry_name = path.get("entry_name").unwrap(); diff --git a/reductstore/src/api/entry/rename_entry.rs b/reductstore/src/api/entry/rename_entry.rs index 169f5bd04..2f2dbb63a 100644 --- a/reductstore/src/api/entry/rename_entry.rs +++ b/reductstore/src/api/entry/rename_entry.rs @@ -25,7 +25,7 @@ pub(crate) async fn rename_entry( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/update_batched.rs b/reductstore/src/api/entry/update_batched.rs index 7c28879c4..c324e9920 100644 --- a/reductstore/src/api/entry/update_batched.rs +++ b/reductstore/src/api/entry/update_batched.rs @@ -32,7 +32,7 @@ pub(crate) async fn update_batched_records( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/update_single.rs b/reductstore/src/api/entry/update_single.rs index c271cd24c..63347607f 100644 --- a/reductstore/src/api/entry/update_single.rs +++ b/reductstore/src/api/entry/update_single.rs @@ -26,14 +26,7 @@ pub(crate) async fn update_record( _: Body, ) -> Result<(), HttpError> { let bucket = path.get("bucket_name").unwrap(); - check_permissions( - &components, - &headers, - WriteAccessPolicy { - bucket: bucket.clone(), - }, - ) - .await?; + check_permissions(&components, &headers, WriteAccessPolicy { bucket }).await?; let ts = parse_timestamp_from_query(¶ms)?; diff --git a/reductstore/src/api/entry/write_batched.rs b/reductstore/src/api/entry/write_batched.rs index c1de31a81..f18aa4951 100644 --- a/reductstore/src/api/entry/write_batched.rs +++ b/reductstore/src/api/entry/write_batched.rs @@ -42,12 +42,12 @@ pub(crate) async fn write_batched_records( Path(path): Path>, body: Body, ) -> Result { - let bucket_name = path.get("bucket_name").unwrap().clone(); + let bucket_name = path.get("bucket_name").unwrap(); check_permissions( &components, &headers, WriteAccessPolicy { - bucket: bucket_name.clone(), + bucket: bucket_name, }, ) .await?; diff --git a/reductstore/src/api/entry/write_single.rs b/reductstore/src/api/entry/write_single.rs index bb341dd75..cf28438ca 100644 --- a/reductstore/src/api/entry/write_single.rs +++ b/reductstore/src/api/entry/write_single.rs @@ -30,14 +30,7 @@ pub(crate) async fn write_record( body: Body, ) -> Result<(), HttpError> { let bucket = path.get("bucket_name").unwrap(); - check_permissions( - &components, - &headers.clone(), - WriteAccessPolicy { - bucket: bucket.clone(), - }, - ) - .await?; + check_permissions(&components, &headers.clone(), WriteAccessPolicy { bucket }).await?; let mut stream = body.into_data_stream(); diff --git a/reductstore/src/api/server/list.rs b/reductstore/src/api/server/list.rs index 5ce15eb37..23c08737e 100644 --- a/reductstore/src/api/server/list.rs +++ b/reductstore/src/api/server/list.rs @@ -25,7 +25,7 @@ pub(crate) async fn list( &components, &headers, ReadAccessPolicy { - bucket: bucket.name.clone(), + bucket: &bucket.name, }, ) .await diff --git a/reductstore/src/auth/policy.rs b/reductstore/src/auth/policy.rs index 835122af7..446be31c5 100644 --- a/reductstore/src/auth/policy.rs +++ b/reductstore/src/auth/policy.rs @@ -1,8 +1,9 @@ -// Copyright 2023 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 use reduct_base::error::ReductError; -use reduct_base::msg::token_api::Token; +use reduct_base::msg::token_api::{Permissions, Token}; +use reduct_base::{forbidden, unprocessable_entity}; /// Policy is a trait that defines the interface for a policy. /// A policy is a set of rules that are applied to a token to determine @@ -46,80 +47,83 @@ pub struct FullAccessPolicy {} impl Policy for FullAccessPolicy { fn validate(&self, token: Result) -> Result<(), ReductError> { - if token? - .permissions - .ok_or(ReductError::internal_server_error("No permissions set"))? - .full_access - { + let token = token?; + if token.permissions.unwrap_or_default().full_access { Ok(()) } else { - Err(ReductError::forbidden("Token doesn't have full access")) + Err(forbidden!( + "Token '{}' doesn't have full access", + token.name + )) } } } /// ReadAccessPolicy validates a token that has read access for a certain bucket -pub struct ReadAccessPolicy { - pub(crate) bucket: String, +pub struct ReadAccessPolicy<'a> { + pub(crate) bucket: &'a str, } -impl Policy for ReadAccessPolicy { +impl Policy for ReadAccessPolicy<'_> { fn validate(&self, token: Result) -> Result<(), ReductError> { - let permissions = &token? - .permissions - .ok_or(ReductError::internal_server_error("No permissions set"))?; + let token = token?; + let permissions = &token.permissions.unwrap_or_default(); if permissions.full_access { return Ok(()); } - for bucket in &permissions.read { - if bucket == &self.bucket { - return Ok(()); - } + if check_bucket_permissions(&permissions.read, self.bucket) { + return Ok(()); } - Err(ReductError::forbidden( - format!( - "Token doesn't have read access for the {} bucket", - self.bucket - ) - .as_str(), + Err(forbidden!( + "Token '{}' doesn't have read access to bucket '{}'", + token.name, + self.bucket )) } } /// WriteAccessPolicy validates a token that has write access for a certain bucket -pub struct WriteAccessPolicy { - pub(crate) bucket: String, +pub struct WriteAccessPolicy<'a> { + pub(crate) bucket: &'a str, } -impl Policy for WriteAccessPolicy { +impl Policy for WriteAccessPolicy<'_> { fn validate(&self, token: Result) -> Result<(), ReductError> { - let permissions = &token? - .permissions - .ok_or(ReductError::internal_server_error("No permissions set"))?; + let token = token?; + let permissions = &token.permissions.unwrap_or_default(); if permissions.full_access { return Ok(()); } - for bucket in &permissions.write { - if bucket == &self.bucket { - return Ok(()); - } + if check_bucket_permissions(&permissions.write, self.bucket) { + return Ok(()); } - Err(ReductError::forbidden( - format!( - "Token doesn't have write access for the {} bucket", - self.bucket - ) - .as_str(), + Err(forbidden!( + "Token '{}' doesn't have write access to bucket '{}'", + token.name, + self.bucket )) } } +fn check_bucket_permissions(token_list: &Vec, bucket: &str) -> bool { + for token_bucket in token_list { + // Check if the token has access for the specified bucket with wildcard support + let wildcard_bucket = token_bucket.ends_with('*'); + if token_bucket == bucket + || (wildcard_bucket && bucket.starts_with(&token_bucket[..token_bucket.len() - 1])) + { + return true; + } + } + false +} + #[cfg(test)] mod tests { use super::*; @@ -129,9 +133,7 @@ mod tests { fn test_anonymous_policy() { let policy = AnonymousPolicy {}; assert!(policy.validate(Ok(Token::default())).is_ok()); - assert!(policy - .validate(Err(ReductError::forbidden("Invalid token"))) - .is_ok()); + assert!(policy.validate(Err(forbidden!("Invalid token"))).is_ok()); } #[test] @@ -139,8 +141,8 @@ mod tests { let policy = AuthenticatedPolicy {}; assert!(policy.validate(Ok(Token::default())).is_ok()); assert_eq!( - policy.validate(Err(ReductError::forbidden("Invalid token"))), - Err(ReductError::forbidden("Invalid token")) + policy.validate(Err(forbidden!("Invalid token"))), + Err(forbidden!("Invalid token")) ); } @@ -148,6 +150,7 @@ mod tests { fn test_full_access_policy() { let policy = FullAccessPolicy {}; let mut token = Token { + name: "test_token".to_string(), permissions: Some(Permissions { full_access: true, read: vec![], @@ -160,21 +163,20 @@ mod tests { token.permissions.as_mut().unwrap().full_access = false; assert_eq!( policy.validate(Ok(token)), - Err(ReductError::forbidden("Token doesn't have full access")) + Err(forbidden!("Token 'test_token' doesn't have full access")) ); assert_eq!( - policy.validate(Err(ReductError::forbidden("Invalid token"))), - Err(ReductError::forbidden("Invalid token")) + policy.validate(Err(forbidden!("Invalid token"))), + Err(forbidden!("Invalid token")) ); } #[test] fn test_read_access_policy() { - let policy = ReadAccessPolicy { - bucket: "bucket".to_string(), - }; + let policy = ReadAccessPolicy { bucket: "bucket" }; let mut token = Token { + name: "test_token".to_string(), permissions: Some(Permissions { full_access: true, read: vec![], @@ -190,24 +192,29 @@ mod tests { token.permissions.as_mut().unwrap().read = vec!["bucket2".to_string()]; assert_eq!( - policy.validate(Ok(token)), - Err(ReductError::forbidden( - "Token doesn't have read access for the bucket bucket" + policy.validate(Ok(token.clone())), + Err(forbidden!( + "Token 'test_token' doesn't have read access to bucket 'bucket'" )) ); + token.permissions.as_mut().unwrap().read = vec!["bucket*".to_string()]; + assert!(policy.validate(Ok(token.clone())).is_ok()); + + token.permissions.as_mut().unwrap().read = vec!["*".to_string()]; + assert!(policy.validate(Ok(token)).is_ok()); + assert_eq!( - policy.validate(Err(ReductError::forbidden("Invalid token"))), - Err(ReductError::forbidden("Invalid token")) + policy.validate(Err(forbidden!("Invalid token"))), + Err(forbidden!("Invalid token")) ); } #[test] fn test_write_access_policy() { - let policy = WriteAccessPolicy { - bucket: "bucket".to_string(), - }; + let policy = WriteAccessPolicy { bucket: "bucket" }; let mut token = Token { + name: "test_token".to_string(), permissions: Some(Permissions { full_access: true, read: vec![], @@ -223,15 +230,21 @@ mod tests { token.permissions.as_mut().unwrap().write = vec!["bucket2".to_string()]; assert_eq!( - policy.validate(Ok(token)), - Err(ReductError::forbidden( - "Token doesn't have write access for the bucket bucket" + policy.validate(Ok(token.clone())), + Err(forbidden!( + "Token 'test_token' doesn't have write access to bucket 'bucket'" )) ); + token.permissions.as_mut().unwrap().write = vec!["bucket*".to_string()]; + assert!(policy.validate(Ok(token.clone())).is_ok()); + + token.permissions.as_mut().unwrap().write = vec!["*".to_string()]; + assert!(policy.validate(Ok(token)).is_ok()); + assert_eq!( - policy.validate(Err(ReductError::forbidden("Invalid token"))), - Err(ReductError::forbidden("Invalid token")) + policy.validate(Err(forbidden!("Invalid token"))), + Err(forbidden!("Invalid token")) ); } } From 8b14954a76d997605049ffaacb624349dbd7d338 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 22 Jul 2025 15:07:22 +0200 Subject: [PATCH 54/93] Check format of read/write token permissions (#881) * check format of read/write permissions * optimize regex * clean code * update CHANGELOG --- CHANGELOG.md | 1 + reductstore/src/auth/policy.rs | 4 +- reductstore/src/auth/token_auth.rs | 4 +- reductstore/src/auth/token_repository.rs | 174 +++++++---------------- 4 files changed, 56 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3251da30d..4728c3a03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for duration literals, [PR-864](https://github.com/reductstore/reductstore/pull/864) - Support for `#ctx_before` and `#ctx_after` directives, [PR-866](https://github.com/reductstore/reductstore/pull/866) - Support for wildcards in write/read token permissions, [PR-877](https://github.com/reductstore/reductstore/pull/877) +- Check format of read/write token permissions, [PR-881](https://github.com/reductstore/reductstore/pull/881) ### Changed diff --git a/reductstore/src/auth/policy.rs b/reductstore/src/auth/policy.rs index 446be31c5..d37382649 100644 --- a/reductstore/src/auth/policy.rs +++ b/reductstore/src/auth/policy.rs @@ -2,8 +2,8 @@ // Licensed under the Business Source License 1.1 use reduct_base::error::ReductError; -use reduct_base::msg::token_api::{Permissions, Token}; -use reduct_base::{forbidden, unprocessable_entity}; +use reduct_base::forbidden; +use reduct_base::msg::token_api::Token; /// Policy is a trait that defines the interface for a policy. /// A policy is a set of rules that are applied to a token to determine diff --git a/reductstore/src/auth/token_auth.rs b/reductstore/src/auth/token_auth.rs index 62e75b814..341c53b0b 100644 --- a/reductstore/src/auth/token_auth.rs +++ b/reductstore/src/auth/token_auth.rs @@ -1,4 +1,4 @@ -// Copyright 2023 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 use crate::auth::policy::Policy; @@ -6,7 +6,7 @@ use crate::auth::token_repository::ManageTokens; use reduct_base::error::ReductError; /// Authorization by token -pub struct TokenAuthorization { +pub(crate) struct TokenAuthorization { api_token: String, } diff --git a/reductstore/src/auth/token_repository.rs b/reductstore/src/auth/token_repository.rs index ad2a7877f..0c41ad1d0 100644 --- a/reductstore/src/auth/token_repository.rs +++ b/reductstore/src/auth/token_repository.rs @@ -1,4 +1,4 @@ -// Copyright 2023-2024 ReductSoftware UG +// Copyright 2023-2025 ReductSoftware UG // Licensed under the Business Source License 1.1 use crate::auth::proto::token::Permissions as ProtoPermissions; @@ -10,19 +10,19 @@ use prost::Message; use prost_wkt_types::Timestamp; use rand::Rng; use reduct_base::error::ReductError; +use reduct_base::msg::token_api::{Permissions, Token, TokenCreateResponse}; use reduct_base::{ bad_request, conflict, internal_server_error, not_found, unauthorized, unprocessable_entity, }; +use regex::Regex; use std::collections::HashMap; use std::path::PathBuf; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use reduct_base::msg::token_api::{Permissions, Token, TokenCreateResponse}; - const TOKEN_REPO_FILE_NAME: &str = ".auth"; const INIT_TOKEN_NAME: &str = "init-token"; -pub trait ManageTokens { +pub(crate) trait ManageTokens { /// Create a new token /// /// # Arguments @@ -39,14 +39,6 @@ pub trait ManageTokens { permissions: Permissions, ) -> Result; - /// Update a token - /// - /// # Arguments - /// - /// `name` - The name of the token - /// `permissions` - The permissions of the token - fn update_token(&mut self, name: &str, permissions: Permissions) -> Result<(), ReductError>; - /// Get a token by name /// /// # Arguments @@ -120,9 +112,10 @@ pub trait ManageTokens { struct TokenRepository { config_path: PathBuf, repo: HashMap, + permission_regex: Regex, } -pub fn parse_bearer_token(authorization_header: &str) -> Result { +pub(crate) fn parse_bearer_token(authorization_header: &str) -> Result { if !authorization_header.starts_with("Bearer ") { return Err(ReductError::unauthorized( "No bearer token in request header", @@ -152,7 +145,7 @@ impl From for ProtoToken { seconds: token.created_at.timestamp(), nanos: token.created_at.timestamp_subsec_nanos() as i32, }), - permissions: permissions, + permissions, } } } @@ -207,7 +200,13 @@ impl TokenRepository { } // Load the token repository from the file system - let mut token_repository = TokenRepository { config_path, repo }; + let permission_regex = + Regex::new(r"^[*a-zA-Z0-9_\-]+$").expect("Invalid regex for permissions"); + let mut token_repository = TokenRepository { + config_path, + repo, + permission_regex, + }; match std::fs::read(&token_repository.config_path) { Ok(data) => { @@ -292,6 +291,15 @@ impl ManageTokens for TokenRepository { return Err(conflict!("Token '{}' already exists", name)); } + for entry in permissions.read.iter().chain(&permissions.write) { + if !self.permission_regex.is_match(entry) { + return Err(unprocessable_entity!( + "Permission can contain only bucket names or wildcard '*', got '{}'", + entry + )); + } + } + let created_at = DateTime::::from(SystemTime::now()); // Create a random hex string @@ -314,24 +322,6 @@ impl ManageTokens for TokenRepository { Ok(TokenCreateResponse { value, created_at }) } - fn update_token(&mut self, name: &str, permissions: Permissions) -> Result<(), ReductError> { - match self.repo.get(name) { - Some(token) => { - if token.is_provisioned { - Err(conflict!("Can't update provisioned token '{}'", name)) - } else { - let mut updated_token = token.clone(); - updated_token.permissions = Some(permissions); - self.repo.insert(name.to_string(), updated_token); - self.save_repo()?; - Ok(()) - } - } - - None => Err(not_found!("Token '{}' doesn't exist", name)), - } - } - fn get_token(&self, name: &str) -> Result<&Token, ReductError> { match self.repo.get(name) { Some(token) => Ok(token), @@ -442,10 +432,6 @@ impl ManageTokens for NoAuthRepository { Err(bad_request!("Authentication is disabled")) } - fn update_token(&mut self, _name: &str, _permissions: Permissions) -> Result<(), ReductError> { - Err(bad_request!("Authentication is disabled")) - } - fn get_token(&self, _name: &str) -> Result<&Token, ReductError> { Err(bad_request!("Authentication is disabled")) } @@ -488,7 +474,7 @@ impl ManageTokens for NoAuthRepository { /// Creates a token repository /// /// If `init_token` is empty, the repository will be stubbed and authentication will be disabled. -pub fn create_token_repository( +pub(crate) fn create_token_repository( path: PathBuf, init_token: &str, ) -> Box { @@ -601,109 +587,51 @@ mod tests { assert_eq!(token, Err(bad_request!("Authentication is disabled"))); } - } - - mod update_token { - use super::*; - #[rstest] - fn test_update_token_ok(mut repo: Box) { - let token = repo - .update_token( - "test", - Permissions { - full_access: false, - read: vec!["test".to_string()], - write: vec![], - }, - ) - .unwrap(); - - assert_eq!(token, ()); - - let token = repo.get_token("test").unwrap().clone(); - - assert_eq!(token.name, "test"); - - let permissions = token.permissions.unwrap(); - assert_eq!(permissions.full_access, false); - assert_eq!(permissions.read, vec!["test".to_string()]); - } #[rstest] - fn test_update_token_not_found(mut repo: Box) { - let token = repo.update_token( + #[case("*", None)] + #[case("bucket_1", None)] + #[case("bucket_2", None)] + #[case("bucket-*", None)] + #[case("%!", Some(unprocessable_entity!("Permission can contain only bucket names or wildcard '*', got '%!'")))] + fn test_create_token_check_format_read( + mut repo: Box, + #[case] bucket: &str, + #[case] expected: Option, + ) { + let token = repo.generate_token( "test-1", Permissions { full_access: true, - read: vec![], + read: vec![bucket.to_string()], write: vec![], }, ); - assert_eq!( - token, - Err(ReductError::not_found("Token 'test-1' doesn't exist")) - ); - } - - #[rstest] - fn test_update_token_persistent(path: PathBuf) { - let mut repo = create_token_repository(path.clone(), "test"); - repo.generate_token( - "test", - Permissions { - full_access: true, - read: vec![], - write: vec![], - }, - ) - .unwrap(); - - repo.update_token( - "test", - Permissions { - full_access: false, - read: vec!["test".to_string()], - write: vec![], - }, - ) - .unwrap(); - - let repo = create_token_repository(path.clone(), "test"); - let token = repo.get_token("test").unwrap().clone(); - - assert_eq!(token.name, "test"); - - let permissions = token.permissions.unwrap(); - assert_eq!(permissions.full_access, false); - assert_eq!(permissions.read, vec!["test".to_string()]); - } - - #[rstest] - fn test_update_provisioned_token(mut repo: Box) { - let token = repo.get_mut_token("test").unwrap(); - token.is_provisioned = true; - - let token = repo.update_token("test", Permissions::default()); - - assert_eq!( - token, - Err(conflict!("Can't update provisioned token 'test'")) - ); + assert_eq!(token.err(), expected); } #[rstest] - fn test_update_token_no_init_token(mut disabled_repo: Box) { - let token = disabled_repo.update_token( - "test", + #[case("*", None)] + #[case("bucket_1", None)] + #[case("bucket_2", None)] + #[case("bucket-*", None)] + #[case("%!", Some(unprocessable_entity!("Permission can contain only bucket names or wildcard '*', got '%!'")))] + fn test_create_token_check_format_write( + mut repo: Box, + #[case] bucket: &str, + #[case] expected: Option, + ) { + let token = repo.generate_token( + "test-1", Permissions { full_access: true, read: vec![], - write: vec![], + write: vec![bucket.to_string()], }, ); - assert_eq!(token, Err(bad_request!("Authentication is disabled"))); + assert_eq!(token.err(), expected); } } From 910564386044496f5723ae86d9e44b53891b59b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 20:54:45 +0200 Subject: [PATCH 55/93] Bump serde_json from 1.0.140 to 1.0.141 (#879) Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.140 to 1.0.141. - [Release notes](https://github.com/serde-rs/json/releases) - [Commits](https://github.com/serde-rs/json/compare/v1.0.140...v1.0.141) --- updated-dependencies: - dependency-name: serde_json dependency-version: 1.0.141 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reduct_base/Cargo.toml | 2 +- reductstore/Cargo.toml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76c7f9701..a9ab60c46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2370,9 +2370,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" dependencies = [ "indexmap", "itoa", diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 2e8ca0e7c..1c315672e 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -27,7 +27,7 @@ all = ["io", "ext"] [dependencies] serde = { version = "1.0.219", features = ["derive"] } -serde_json = { version = "1.0.140", features = ["preserve_order"] } +serde_json = { version = "1.0.141", features = ["preserve_order"] } int-enum = "0.5.0" chrono = { version = "0.4.41", features = ["serde"] } url = "2.5.4" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 63c7a63cf..0834800fc 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -38,7 +38,7 @@ hex = "0.4.3" prost-wkt-types = "0.6.1" rand = "0.9.1" serde = { version = "1.0.219", features = ["derive"] } -serde_json = "1.0.140" +serde_json = "1.0.141" regex = "1.11.1" bytes = "1.10.1" axum = { version = "0.8.4", features = ["default", "macros"] } @@ -69,7 +69,7 @@ prost = "0.13.1" prost-build = "0.13.1" reqwest = { version = "0.12.22", default-features = false, features = ["rustls-tls", "blocking", "json"] } chrono = "0.4.41" -serde_json = "1.0.140" +serde_json = "1.0.141" [dev-dependencies] mockall = "0.13.1" From 8f0c17fc614dc7941a3187eab4d672e9fbece6dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 20:55:03 +0200 Subject: [PATCH 56/93] Bump rand from 0.9.1 to 0.9.2 (#878) Bumps [rand](https://github.com/rust-random/rand) from 0.9.1 to 0.9.2. - [Release notes](https://github.com/rust-random/rand/releases) - [Changelog](https://github.com/rust-random/rand/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/rand/compare/rand_core-0.9.1...rand_core-0.9.2) --- updated-dependencies: - dependency-name: rand dependency-version: 0.9.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- reductstore/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9ab60c46..bdb18ca37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1963,9 +1963,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha", "rand_core", diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index 0834800fc..d32afb4f4 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -36,7 +36,7 @@ zip = "4.3.0" tempfile = "3.20.0" hex = "0.4.3" prost-wkt-types = "0.6.1" -rand = "0.9.1" +rand = "0.9.2" serde = { version = "1.0.219", features = ["derive"] } serde_json = "1.0.141" regex = "1.11.1" From 4e291c39444c9180316519b699c16226b320dae2 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 24 Jul 2025 22:42:29 +0200 Subject: [PATCH 57/93] Update ROS extension up to v0.2.0 (#882) * update ROS extenstion up to v0.2.0 * update CHANGELOG --- CHANGELOG.md | 3 ++- reductstore/build.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4728c3a03..6aac9ca61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) - Run all operands after a compute-staged one on the compute stage, [PR-835](https://github.com/reductstore/reductstore/pull/835) - Replace auto-staging for extension filtering with when condition in ext parameter, [PR-838](https://github.com/reductstore/reductstore/pull/838) -- Update ReductSelect to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) +- Update ReductSelect up to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) +- Update ReductROS up to v0.2.0 with binary data encoding, [PR-882]https://github.com/reductstore/reductstore/pull/882) ### Fixed diff --git a/reductstore/build.rs b/reductstore/build.rs index eea1aa4dd..862fc317e 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -33,7 +33,7 @@ fn main() -> Result<(), Box> { download_ext("select-ext", "v0.3.0"); #[cfg(feature = "ros-ext")] - download_ext("ros-ext", "v0.1.0"); + download_ext("ros-ext", "v0.2.0"); // get build time and commit let build_time = chrono::DateTime::::from(SystemTime::now()) From b8731c385c0fa98f93fff508d0321cfd39fe4fb5 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 24 Jul 2025 22:42:59 +0200 Subject: [PATCH 58/93] fix CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aac9ca61..70129fa99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Run all operands after a compute-staged one on the compute stage, [PR-835](https://github.com/reductstore/reductstore/pull/835) - Replace auto-staging for extension filtering with when condition in ext parameter, [PR-838](https://github.com/reductstore/reductstore/pull/838) - Update ReductSelect up to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) -- Update ReductROS up to v0.2.0 with binary data encoding, [PR-882]https://github.com/reductstore/reductstore/pull/882) +- Update ReductROS up to v0.2.0 with binary data encoding, [PR-882](https://github.com/reductstore/reductstore/pull/882) ### Fixed From 5eb17f550f96143148a84a0c8e6efd82e63639ce Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 24 Jul 2025 23:31:16 +0200 Subject: [PATCH 59/93] update web console up to v1.11.0 (#883) --- CHANGELOG.md | 1 + reductstore/build.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70129fa99..92518ccdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Replace auto-staging for extension filtering with when condition in ext parameter, [PR-838](https://github.com/reductstore/reductstore/pull/838) - Update ReductSelect up to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) - Update ReductROS up to v0.2.0 with binary data encoding, [PR-882](https://github.com/reductstore/reductstore/pull/882) +- Update WebConsole up to v1.11.0 with many improvements, [PR-883](https://github.com/reductstore/reductstore/pull/883) ### Fixed diff --git a/reductstore/build.rs b/reductstore/build.rs index 862fc317e..c4ac0242b 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -27,7 +27,7 @@ fn main() -> Result<(), Box> { .expect("Failed to compile protos"); #[cfg(feature = "web-console")] - download_web_console("v1.10.2"); + download_web_console("v1.11.0"); #[cfg(feature = "select-ext")] download_ext("select-ext", "v0.3.0"); From c6be8eb412d65c63a5fb215ed97a282cf710194d Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Thu, 24 Jul 2025 23:58:33 +0200 Subject: [PATCH 60/93] fix api test for ros extension --- integration_tests/api/ext_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/api/ext_test.py b/integration_tests/api/ext_test.py index d397bd4fc..a1a574563 100644 --- a/integration_tests/api/ext_test.py +++ b/integration_tests/api/ext_test.py @@ -65,6 +65,6 @@ def test__ros_ext(base_url, bucket, session): assert ( resp.headers["x-reduct-time-24"] - == "16,application/json,encoding=cdr,schema=std_msgs/String,topic=/test" + == "16,application/json,@encoding=cdr,@schema=std_msgs/String,@topic=/test" ) assert resp.content == b'{"data":"hello"}' From 408b446a6c6f68e135c9d70da47c4a73f7251779 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 20:15:41 +0200 Subject: [PATCH 61/93] Bump tokio from 1.46.1 to 1.47.0 (#884) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.46.1 to 1.47.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.46.1...tokio-1.47.0) --- updated-dependencies: - dependency-name: tokio dependency-version: 1.47.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 24 +++++++++++++++++------- reduct_base/Cargo.toml | 2 +- reductstore/Cargo.toml | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdb18ca37..4b3e591e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1065,7 +1065,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -1904,7 +1904,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2", + "socket2 0.5.10", "thiserror", "tokio", "tracing", @@ -1941,7 +1941,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.5.10", "tracing", "windows-sys 0.59.0", ] @@ -2503,6 +2503,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2695,9 +2705,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" dependencies = [ "backtrace", "bytes", @@ -2708,9 +2718,9 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 1c315672e..51ef46239 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -34,7 +34,7 @@ url = "2.5.4" http = "1.2.0" bytes = "1.10.0" async-trait = { version = "0.1.87" , optional = true } -tokio = { version = "1.46.1", optional = true, features = ["default", "rt", "time"] } +tokio = { version = "1.47.0", optional = true, features = ["default", "rt", "time"] } log = "0.4.0" thread-id = "5.0.0" futures = "0.3.31" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index d32afb4f4..fd9e242cf 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -43,7 +43,7 @@ regex = "1.11.1" bytes = "1.10.1" axum = { version = "0.8.4", features = ["default", "macros"] } axum-extra = { version = "0.10.0", features = ["default", "typed-header"] } -tokio = { version = "1.46.1", features = ["full"] } +tokio = { version = "1.47.0", features = ["full"] } hyper = { version = "1.6.0", features = ["full"] } futures-util = "0.3.31" axum-server = { version = "0.7.1", features = ["tls-rustls"] } From 648703dff43d955bb13db1a6391c02b615d92318 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 20:24:35 +0200 Subject: [PATCH 62/93] Bump rstest from 0.25.0 to 0.26.1 (#885) Bumps [rstest](https://github.com/la10736/rstest) from 0.25.0 to 0.26.1. - [Release notes](https://github.com/la10736/rstest/releases) - [Changelog](https://github.com/la10736/rstest/blob/master/CHANGELOG.md) - [Commits](https://github.com/la10736/rstest/compare/v0.25.0...v0.26.1) --- updated-dependencies: - dependency-name: rstest dependency-version: 0.26.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 9 ++++----- reduct_base/Cargo.toml | 2 +- reductstore/Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b3e591e6..1d36ccf2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2181,21 +2181,20 @@ dependencies = [ [[package]] name = "rstest" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", "rstest_macros", - "rustc_version", ] [[package]] name = "rstest_macros" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", diff --git a/reduct_base/Cargo.toml b/reduct_base/Cargo.toml index 51ef46239..228b8b6d5 100644 --- a/reduct_base/Cargo.toml +++ b/reduct_base/Cargo.toml @@ -40,4 +40,4 @@ thread-id = "5.0.0" futures = "0.3.31" [dev-dependencies] -rstest = "0.25.0" +rstest = "0.26.1" diff --git a/reductstore/Cargo.toml b/reductstore/Cargo.toml index fd9e242cf..779a86833 100644 --- a/reductstore/Cargo.toml +++ b/reductstore/Cargo.toml @@ -73,7 +73,7 @@ serde_json = "1.0.141" [dev-dependencies] mockall = "0.13.1" -rstest = "0.25.0" +rstest = "0.26.1" serial_test = "3.2.0" test-log = "0.2.18" reqwest = { version = "0.12.22", default-features = false, features = ["rustls-tls", "blocking"] } From 7c15fd0b445381b893b00fc761edfc41cceec564 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 29 Jul 2025 10:03:10 +0200 Subject: [PATCH 63/93] Update ReductSelect up to v0.4.0 (#889) * update select-ext up to v0.4.0 * update CHANGELOG --- CHANGELOG.md | 1 + reductstore/build.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92518ccdc..477eaa6ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update ReductSelect up to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) - Update ReductROS up to v0.2.0 with binary data encoding, [PR-882](https://github.com/reductstore/reductstore/pull/882) - Update WebConsole up to v1.11.0 with many improvements, [PR-883](https://github.com/reductstore/reductstore/pull/883) +- Update ReductSelect up to v0.4.0, [PR-889](https://github.com/reductstore/reductstore/pull/889) ### Fixed diff --git a/reductstore/build.rs b/reductstore/build.rs index c4ac0242b..15534f103 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -30,7 +30,7 @@ fn main() -> Result<(), Box> { download_web_console("v1.11.0"); #[cfg(feature = "select-ext")] - download_ext("select-ext", "v0.3.0"); + download_ext("select-ext", "v0.4.0"); #[cfg(feature = "ros-ext")] download_ext("ros-ext", "v0.2.0"); From 0b2f8531e28a9e2a9fde1e490bb34ef505861266 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Tue, 29 Jul 2025 13:19:07 +0200 Subject: [PATCH 64/93] update webconsole -> v1.11.1 and select-ext -> 0.4.1 (#890) --- CHANGELOG.md | 1 + reductstore/build.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 477eaa6ff..e40c92c15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update ReductROS up to v0.2.0 with binary data encoding, [PR-882](https://github.com/reductstore/reductstore/pull/882) - Update WebConsole up to v1.11.0 with many improvements, [PR-883](https://github.com/reductstore/reductstore/pull/883) - Update ReductSelect up to v0.4.0, [PR-889](https://github.com/reductstore/reductstore/pull/889) +- Update Web Console up to v1.11.1 and ReductSelect up to 0.4.1, [PR-890](https://github.com/reductstore/reductstore/pull/890) ### Fixed diff --git a/reductstore/build.rs b/reductstore/build.rs index 15534f103..ebd137cde 100644 --- a/reductstore/build.rs +++ b/reductstore/build.rs @@ -27,10 +27,10 @@ fn main() -> Result<(), Box> { .expect("Failed to compile protos"); #[cfg(feature = "web-console")] - download_web_console("v1.11.0"); + download_web_console("v1.11.1"); #[cfg(feature = "select-ext")] - download_ext("select-ext", "v0.4.0"); + download_ext("select-ext", "v0.4.1"); #[cfg(feature = "ros-ext")] download_ext("ros-ext", "v0.2.0"); From 7abf23ff00a38ccbd0e5f5a66a310187ee2c7edd Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 30 Jul 2025 11:03:42 +0200 Subject: [PATCH 65/93] Fix query finalization in ExtRepository (#891) * refactor ext repo to keep nocontent error * fix tests * update CHANGELOG * add tests --- CHANGELOG.md | 1 + reductstore/src/api/entry/read_batched.rs | 55 ++++++------ reductstore/src/ext/ext_repository.rs | 94 +++++++++++--------- reductstore/src/ext/ext_repository/create.rs | 12 +-- 4 files changed, 86 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e40c92c15..00398bab6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix hanging query request if no extension registered, [PR-830](https://github.com/reductstore/reductstore/pull/830) - Fix `$limit` operator in extension context, [PR-848](https://github.com/reductstore/reductstore/pull/848) +- Fix query finalization in ExtRepository, [PR-891](https://github.com/reductstore/reductstore/pull/891) ### Removed diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index e9de749bd..bc1e099fe 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -130,7 +130,7 @@ async fn fetch_and_response_batched_records( let start_time = std::time::Instant::now(); loop { - let reader = match next_record_reader( + let batch_of_readers = match next_record_reader( query_id, rx.upgrade()?, &format!("{}/{}/{}", bucket_name, entry_name, query_id), @@ -143,9 +143,9 @@ async fn fetch_and_response_batched_records( None => continue, }; - match reader { - Ok(next_readers) => { - for reader in next_readers { + for reader in batch_of_readers { + match reader { + Ok(reader) => { { let (name, value) = make_batch_header(&reader); header_size += name.as_str().len() + value.to_str().unwrap().len() + 2; @@ -155,28 +155,28 @@ async fn fetch_and_response_batched_records( readers.push(reader); } - - if header_size > io_settings.batch_max_metadata_size - || body_size > io_settings.batch_max_size - || readers.len() > io_settings.batch_max_records - || start_time.elapsed() > io_settings.batch_timeout - { - break; - } - } - Err(err) => { - if readers.is_empty() { - return Err(HttpError::from(err)); - } else { - if err.status() == ErrorCode::NoContent { - last = true; - break; - } else { + Err(err) => { + if readers.is_empty() { return Err(HttpError::from(err)); + } else { + if err.status() == ErrorCode::NoContent { + last = true; + break; + } else { + return Err(HttpError::from(err)); + } } } } - }; + } + + if header_size > io_settings.batch_max_metadata_size + || body_size > io_settings.batch_max_size + || readers.len() > io_settings.batch_max_records + || start_time.elapsed() > io_settings.batch_timeout + { + break; + } } // TODO: it's workaround @@ -212,7 +212,7 @@ async fn next_record_reader( query_path: &str, recv_timeout: Duration, ext_repository: &BoxedManageExtensions, -) -> Option, ReductError>> { +) -> Option>> { // we need to wait for the first record if let Ok(result) = timeout( recv_timeout, @@ -487,10 +487,11 @@ mod tests { ) .await .unwrap() - .unwrap() - .err() - .unwrap() - .status(), + .unwrap()[0] + .as_ref() + .err() + .unwrap() + .status(), ErrorCode::NoContent, "should return None if the query is closed" ); diff --git a/reductstore/src/ext/ext_repository.rs b/reductstore/src/ext/ext_repository.rs index da81a49ee..57d0b586a 100644 --- a/reductstore/src/ext/ext_repository.rs +++ b/reductstore/src/ext/ext_repository.rs @@ -58,7 +58,7 @@ pub(crate) trait ManageExtensions { &self, query_id: u64, query_rx: Arc>, - ) -> Option, ReductError>>; + ) -> Option>>; } pub type BoxedManageExtensions = Box; @@ -190,7 +190,7 @@ impl ManageExtensions for ExtRepository { &self, query_id: u64, query_rx: Arc>, - ) -> Option, ReductError>> { + ) -> Option>> { // TODO: The code is awkward, we need to refactor it // unfortunately stream! macro does not work here and crashes compiler let mut lock = self.query_map.write().await; @@ -206,10 +206,15 @@ impl ManageExtensions for ExtRepository { if result.is_none() { // If no record is available, return a no content error to finish the query. - return Some(Err(no_content!("No content"))); + return Some(vec![Err(no_content!("No content"))]); } - return result; + return result.map(|r| { + r.map_or_else( + |e| vec![Err(e)], + |records| records.into_iter().map(Ok).collect(), + ) + }); } }; @@ -221,7 +226,7 @@ impl ManageExtensions for ExtRepository { if let Some(result) = item { if let Err(e) = result { - return Some(Err(e)); + return Some(vec![Err(e)]); } let record = result.unwrap(); @@ -231,24 +236,21 @@ impl ManageExtensions for ExtRepository { let mut commited_records = vec![]; for record in records { if let Some(rec) = query.commiter.commit_record(record).await { - match rec { - Ok(rec) => commited_records.push(rec), - Err(e) => return Some(Err(e)), - } + commited_records.push(rec); } } if commited_records.is_empty() { None } else { - Some(Ok(commited_records)) + Some(commited_records) } } Ok(None) => { query.current_stream = None; None } - Err(e) => Some(Err(e)), + Err(e) => Some(vec![Err(e)]), }; } else { // stream is empty, we need to process the next record @@ -257,7 +259,7 @@ impl ManageExtensions for ExtRepository { } let Some(record) = query_rx.write().await.recv().await else { - return Some(Err(no_content!("No content"))); + return Some(vec![Err(no_content!("No content"))]); }; let record = match record { @@ -267,15 +269,15 @@ impl ManageExtensions for ExtRepository { if let Some(last_record) = query.commiter.flush().await { match last_record { Ok(rec) => { - Some(Ok(vec![rec])) // return the last record if available + Some(vec![Ok(rec), Err(e)]) // return the last record if available and the error } - Err(e) => Some(Err(e)), + Err(e) => Some(vec![Err(e)]), } } else { - Some(Err(e)) // return no content error if no last record + Some(vec![Err(e)]) } } else { - Some(Err(e)) + Some(vec![Err(e)]) }; } }; @@ -284,7 +286,7 @@ impl ManageExtensions for ExtRepository { let stream = match query.processor.process_record(Box::new(record)).await { Ok(stream) => stream, - Err(e) => return Some(Err(e)), + Err(e) => return Some(vec![Err(e)]), }; query.current_stream = Some(Box::into_pin(stream)); @@ -519,10 +521,11 @@ pub(super) mod tests { let query_rx = Arc::new(AsyncRwLock::new(rx)); assert_eq!( - mocked_ext_repo + *mocked_ext_repo .fetch_and_process_record(1, query_rx) .await - .unwrap() + .unwrap()[0] + .as_ref() .err() .unwrap(), no_content!("No content"), @@ -540,10 +543,11 @@ pub(super) mod tests { let query_rx = Arc::new(AsyncRwLock::new(rx)); assert_eq!( - mocked_ext_repo + *mocked_ext_repo .fetch_and_process_record(1, query_rx) .await - .unwrap() + .unwrap()[0] + .as_ref() .err() .unwrap(), err @@ -562,7 +566,8 @@ pub(super) mod tests { assert!(mocked_ext_repo .fetch_and_process_record(1, query_rx) .await - .unwrap() + .unwrap()[0] + .as_ref() .is_ok(),); } @@ -665,19 +670,19 @@ pub(super) mod tests { let mut records = mocked_ext_repo .fetch_and_process_record(1, query_rx.clone()) .await - .unwrap() .unwrap(); assert_eq!(records.len(), 1, "Should return one record"); - let record = records.first_mut().unwrap(); + let record = records.get_mut(0).unwrap().as_mut().unwrap(); assert_eq!(record.read().await, None); assert_eq!( - mocked_ext_repo + *mocked_ext_repo .fetch_and_process_record(1, query_rx) .await - .unwrap() + .unwrap()[0] + .as_ref() .err() .unwrap(), no_content!("") @@ -748,25 +753,24 @@ pub(super) mod tests { "we don't commit the record waiting for flush" ); + let results = mocked_ext_repo + .fetch_and_process_record(1, query_rx.clone()) + .await + .unwrap(); + + assert_eq!( + results.len(), + 2, + "Should return one record and non-content error" + ); assert!( - mocked_ext_repo - .fetch_and_process_record(1, query_rx.clone()) - .await - .unwrap() - .is_ok(), + results[0].as_ref().is_ok(), "we should get the record from flush" ); - - drop(tx); // close the channel to simulate no more records assert_eq!( - mocked_ext_repo - .fetch_and_process_record(1, query_rx) - .await - .unwrap() - .err() - .unwrap(), - no_content!("No content"), - "and we are done" + results[1].as_ref().err().unwrap().status(), + NoContent, + "we should get no content error" ); } } @@ -828,7 +832,8 @@ pub(super) mod tests { mocked_ext_repo .fetch_and_process_record(1, query_rx.clone()) .await - .unwrap() + .unwrap()[0] + .as_ref() .expect("Should return a record"); assert!( @@ -840,10 +845,11 @@ pub(super) mod tests { ); assert_eq!( - mocked_ext_repo + *mocked_ext_repo .fetch_and_process_record(1, query_rx) .await - .unwrap() + .unwrap()[0] + .as_ref() .err() .unwrap(), no_content!("") diff --git a/reductstore/src/ext/ext_repository/create.rs b/reductstore/src/ext/ext_repository/create.rs index 29a5fff8d..e5c741a38 100644 --- a/reductstore/src/ext/ext_repository/create.rs +++ b/reductstore/src/ext/ext_repository/create.rs @@ -56,17 +56,17 @@ pub fn create_ext_repository( &self, _query_id: u64, query_rx: Arc>, - ) -> Option, ReductError>> { + ) -> Option>> { let result = query_rx .write() .await .recv() .await - .map(|record| record.map(|r| vec![Box::new(r) as BoxedReadRecord])); + .map(|record| vec![record.map(|r| Box::new(r) as BoxedReadRecord)]); if result.is_none() { // If no record is available, return a no content error to finish the query. - return Some(Err(no_content!(""))); + return Some(vec![Err(no_content!("No content"))]); } result @@ -109,7 +109,9 @@ mod tests { result.is_some(), "Should return Some if no record is available" ); - let err = result.unwrap().err().unwrap(); - assert_eq!(err.status(), NoContent); + assert_eq!( + result.unwrap()[0].as_ref().err().unwrap().status(), + NoContent + ); } } From db4fb2179beac2aa4cbf592001917560fd864f48 Mon Sep 17 00:00:00 2001 From: Alexey Timin Date: Wed, 30 Jul 2025 14:41:43 +0200 Subject: [PATCH 66/93] fix last record in continuous query (#893) --- reductstore/src/api/entry/read_batched.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/reductstore/src/api/entry/read_batched.rs b/reductstore/src/api/entry/read_batched.rs index bc1e099fe..955f4056e 100644 --- a/reductstore/src/api/entry/read_batched.rs +++ b/reductstore/src/api/entry/read_batched.rs @@ -130,7 +130,7 @@ async fn fetch_and_response_batched_records( let start_time = std::time::Instant::now(); loop { - let batch_of_readers = match next_record_reader( + let batch_of_readers = match next_record_readers( query_id, rx.upgrade()?, &format!("{}/{}/{}", bucket_name, entry_name, query_id), @@ -170,6 +170,10 @@ async fn fetch_and_response_batched_records( } } + if last { + break; + } + if header_size > io_settings.batch_max_metadata_size || body_size > io_settings.batch_max_size || readers.len() > io_settings.batch_max_records @@ -206,7 +210,7 @@ async fn fetch_and_response_batched_records( // This function is used to get the next record from the query receiver // created for better testing -async fn next_record_reader( +async fn next_record_readers( query_id: u64, rx: Arc>, query_path: &str, @@ -453,7 +457,7 @@ mod tests { assert!( timeout( Duration::from_secs(1), - next_record_reader( + next_record_readers( 1, rx.clone(), "", @@ -477,7 +481,7 @@ mod tests { assert_eq!( timeout( Duration::from_secs(1), - next_record_reader( + next_record_readers( 1, rx.clone(), "", From 8396db8bb532884196ea07c0f7d2ee54201b7873 Mon Sep 17 00:00:00 2001 From: Anthony Date: Wed, 30 Jul 2025 17:11:28 +0200 Subject: [PATCH 67/93] 880 feature as a user can select labels which i need in the query response (#888) * parse select labels directive * add test parsing select labels directive * implement set_labels and filter labels * fix usage of labels after when filter * resolve merge conflict * test select labels filter * test setting labels on filter record trait * test directive parsing error cases * test for setting labels * test setting labels for filter record implementation * Test that we can modify the metadata through meta_mut() * changelog select_labels directive --------- Co-authored-by: Alexey Timin --- CHANGELOG.md | 97 +---------- misc/test_script.py | 39 +++++ reduct_base/src/io.rs | 25 +++ reductstore/src/api/entry/read_batched.rs | 4 +- reductstore/src/ext/ext_repository.rs | 4 + reductstore/src/ext/filter.rs | 19 +++ .../src/replication/transaction_filter.rs | 26 +++ reductstore/src/storage/block_manager.rs | 10 +- .../src/storage/entry/io/record_reader.rs | 31 +++- reductstore/src/storage/entry/read_record.rs | 2 +- reductstore/src/storage/query.rs | 1 + .../src/storage/query/condition/parser.rs | 149 ++++++++++++++--- reductstore/src/storage/query/filters.rs | 8 + reductstore/src/storage/query/filters/when.rs | 155 +++++++++++++++++- reductstore/src/storage/query/historical.rs | 35 +++- 15 files changed, 469 insertions(+), 136 deletions(-) create mode 100644 misc/test_script.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 00398bab6..e3250e8e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for `#ctx_before` and `#ctx_after` directives, [PR-866](https://github.com/reductstore/reductstore/pull/866) - Support for wildcards in write/read token permissions, [PR-877](https://github.com/reductstore/reductstore/pull/877) - Check format of read/write token permissions, [PR-881](https://github.com/reductstore/reductstore/pull/881) +- Add support for selecting specific labels via the `#select_labels` directive, [PR-888](https://github.com/reductstore/reductstore/pull/888) ### Changed @@ -68,7 +69,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix lock of write channel for small chunks, [PR-834](https://github.com/reductstore/reductstore/pull/834) - ## [1.15.2] - 2025-05-21 ### Changed @@ -99,7 +99,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - RS-645: Implement `$timestamp` operator, [PR-798](https://github.com/reductstore/reductstore/pull/798) - RS-646: Enable logging in extensions, [PR-646](https://github.com/reductstore/reductstore/pull/794) - ### Changed - Minimum Rust version to 1.85 @@ -242,7 +241,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update Web Console up to v1.8.0, [PR-655](https://github.com/reductstore/reductstore/pull/655) - ### Fixed - RS-544: Fix keeping quota error, [PR-654](https://github.com/reductstore/reductstore/pull/654) @@ -255,7 +253,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - RS-536: Update README.md, [PR-649](https://github.com/reductstore/reductstore/pull/649) - RS-193: Cross-compilation in CI/CD, [PR-651](https://github.com/reductstore/reductstore/pull/651) - ## [1.12.4] - 2024-11-20 ### Fixed @@ -744,7 +741,7 @@ reduct-rs: `ReductClient.url`, `ReductClient.token`, `ReductCientBuilder.try_bui ### Added -- Labels for `POST|GET /api/v1/:bucket/:entry` as headers with +- Labels for `POST|GET /api/v1/:bucket/:entry` as headers with prefix `x-reduct-label-`, [PR-224](https://github.com/reductstore/reductstore/pull/224) - `include-