diff --git a/.github/actions/run-minio/action.yml b/.github/actions/run-minio/action.yml new file mode 100644 index 000000000..42fb34e79 --- /dev/null +++ b/.github/actions/run-minio/action.yml @@ -0,0 +1,36 @@ +name: run-minio +author: atimin +description: "Run MinIO server and provision a bucket" +inputs: + access_key: + description: "MinIO access key" + required: true + default: "minioadmin" + secret_key: + description: "MinIO secret key" + required: true + default: "minioadmin" + bucket_name: + description: "Name of the bucket to create" + required: true + default: "test-bucket" +runs: + using: "composite" + steps: + - name: Set up MinIO server + run: | + docker run -d --name minio-server -p 9000:9000 \ + -e "MINIO_ROOT_USER=${{ inputs.access_key }}" \ + -e "MINIO_ROOT_PASSWORD=${{ inputs.secret_key }}" \ + minio/minio server /data + sleep 5 # Wait for the server to start + docker logs minio-server + shell: bash + + - name: Create bucket + run: | + echo "Create bucket ${{ inputs.bucket_name }}" + + docker exec minio-server mc alias set local http://localhost:9000 ${{ inputs.access_key }} ${{ inputs.secret_key }} + docker exec minio-server mc mb local/${{ inputs.bucket_name }} + shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9c009edb..d1415cfc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: ci on: push: - branches: [main, stable] + branches: [ main, stable ] tags: - "v*" paths-ignore: @@ -10,7 +10,7 @@ on: - CHANGELOG.md pull_request: - branches: [main, stable] + branches: [ main, stable ] paths-ignore: - docs/** - README.md @@ -18,11 +18,9 @@ on: env: REGISTRY_IMAGE: reduct/store - MINIMAL_RUST_VERSION: 1.85.0 jobs: rust_fmt: - runs-on: ubuntu-latest name: Rust Linter steps: @@ -43,8 +41,12 @@ jobs: uses: docker/build-push-action@v4 with: context: . + file: buildx.Dockerfile tags: ${{github.repository}}:latest outputs: type=docker,dest=/tmp/image.tar + build-args: | + ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} + RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} - name: Upload artifact uses: actions/upload-artifact@v4 @@ -94,7 +96,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: ${{ env.MINIMAL_RUST_VERSION }} + toolchain: ${{ vars.MINIMAL_RUST_VERSION }} - name: Install toolchain run: rustup target add ${{ matrix.target }} @@ -118,7 +120,8 @@ jobs: - name: Build binary env: RUSTFLAGS: "-C target-feature=+crt-static" - run: cargo build --release -p reductstore --target ${{ matrix.target }} + ARTIFACT_SAS_URL: ${{ secrets.ARTIFACT_SAS_URL }} + run: cargo build --release -p reductstore --target ${{ matrix.target }} --all-features - name: Upload binary uses: actions/upload-artifact@v4 @@ -142,19 +145,20 @@ jobs: - build_binaries strategy: matrix: - os: [ubuntu-24.04, windows-2022, macos-13] + os: [ ubuntu-24.04, windows-2022, macos-14 ] include: - os: ubuntu-24.04 target: x86_64-unknown-linux-gnu - os: windows-2022 target: x86_64-pc-windows-gnu - - os: macos-13 - target: x86_64-apple-darwin + - os: macos-14 + target: aarch64-apple-darwin runs-on: ${{ matrix.os }} timeout-minutes: 5 env: RUST_BACKTRACE: 1 + RUST_LOG: debug steps: - uses: actions/download-artifact@v4 with: @@ -183,7 +187,11 @@ jobs: - rust_fmt env: CARGO_TERM_COLOR: always - RUST_LOG: trace # expand logging + RUST_LOG: debug # expand logging + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_BUCKET: test + MINIO_ENDPOINT: http://127.0.0.1:9000 steps: - uses: actions/checkout@v4 @@ -198,8 +206,16 @@ jobs: version: "26.x" repo-token: ${{ secrets.ACTION_GITHUB_TOKEN }} + - uses: ./.github/actions/run-minio + with: + access_key: ${{ env.MINIO_ACCESS_KEY }} + secret_key: ${{ env.MINIO_SECRET_KEY }} + bucket_name: ${{ env.MINIO_BUCKET }} + - name: Generate code coverage - run: cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info + env: + ARTIFACT_SAS_URL: ${{ secrets.ARTIFACT_SAS_URL }} + run: cargo llvm-cov --features s3-backend,fs-backend,ci --workspace --lcov --output-path lcov.info - name: Upload coverage to Codecov continue-on-error: true @@ -217,15 +233,20 @@ jobs: - build strategy: matrix: - token: ["", "XXXX"] - cert_path: ["", "/misc/certificate.crt"] - license_path: ["", "/misc/license.lic"] + token: [ "", "XXXX" ] + cert_path: [ "", "/misc/certificate.crt" ] + license_path: [ "", "/misc/license.lic" ] + backend: [ "fs", "s3" ] include: - cert_path: "/misc/certificate.crt" url: https://127.0.0.1:8383 - cert_path: "" url: http://127.0.0.1:8383 timeout-minutes: 5 + env: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_BUCKET: test steps: - uses: actions/checkout@v4 @@ -243,7 +264,8 @@ jobs: - name: Create license run: echo '${{secrets.LICENSE_KEY}}' > ./misc/license.lic - - name: Run Database + - name: Run ReductStore with FileSystem backend + if: ${{matrix.backend == 'fs'}} run: | docker run --network=host -v ${PWD}/misc:/misc --env RS_API_TOKEN=${{matrix.token}} \ --name reduct \ @@ -254,6 +276,31 @@ jobs: -d ${{github.repository}} sleep 5 + - name: Run MinIO server and create bucket + if : ${{matrix.backend == 's3'}} + uses: ./.github/actions/run-minio + with: + access_key: ${{ env.MINIO_ACCESS_KEY }} + secret_key: ${{ env.MINIO_SECRET_KEY }} + bucket_name: ${{ env.MINIO_BUCKET }} + + - name: Run ReductStore with S3 backend + if: ${{matrix.backend == 's3'}} + run: | + docker run --network=host -v ${PWD}/misc:/misc \ + --env RS_API_TOKEN=${{matrix.token}} \ + --env RS_CERT_PATH=${{matrix.cert_path}} \ + --env RS_LICENSE_PATH=${{matrix.license_path}} \ + --env RS_CERT_KEY_PATH=/misc/privateKey.key \ + --env RS_STORAGE_TYPE=s3 \ + --env RS_REMOTE_BACKEND_ENDPOINT=${{ env.MINIO_ENDPOINT }} \ + --env RS_REMOTE_ACCESS_KEY=${{ env.MINIO_ACCESS_KEY }} \ + --env RS_REMOTE_SECRET_KEY=${{ env.MINIO_SECRET_KEY }} \ + --env RS_REMOTE_BUCKET=${{ env.MINIO_BUCKET }} \ + --env RS_CORS_ALLOW_ORIGIN="https://first-allowed-origin.com, https://second-allowed-origin.com" \ + --name reduct -d ${{github.repository}} + sleep 5 + - name: Build API tests run: | docker login -u ${{ secrets.DOCKER_USER }} -p ${{ secrets.DOCKER_TOKEN }} @@ -280,7 +327,8 @@ jobs: version: [ "latest", - "v1.14.0", + "v1.15.6", + "v1.14.8", "v1.13.5", "v1.12.4", "v1.11.2", @@ -360,12 +408,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - cmd: ["stop", "kill"] + backend: [ "fs", "s3" ] + cmd: [ "stop", "kill" ] needs: - unit_tests - build env: RS_API_TOKEN: XXXX + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + MINIO_BUCKET: test steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -383,12 +435,33 @@ jobs: docker load --input /tmp/image.tar docker image ls -a - - name: Run ReductStore + - name: Run ReductStore with FileSystem backend + if: ${{matrix.backend == 'fs'}} run: | docker run --network=host -v ./data:/data --env RS_API_TOKEN=${RS_API_TOKEN} --name reductstore -d ${{github.repository}} sleep 5 docker logs reductstore + - name: Run MiniO server and create bucket + if : ${{matrix.backend == 's3'}} + uses: ./.github/actions/run-minio + with: + access_key: ${{ env.MINIO_ACCESS_KEY }} + secret_key: ${{ env.MINIO_SECRET_KEY }} + bucket_name: ${{ env.MINIO_BUCKET }} + + - name: Run ReductStore with S3 backend + if: ${{matrix.backend == 's3'}} + run: | + docker run --network=host -v ./data:/data \ + --env RS_API_TOKEN=${RS_API_TOKEN} \ + --env RS_STORAGE_TYPE=s3 \ + --env RS_REMOTE_BACKEND_ENDPOINT=http://127.0.0.1:9000 \ + --env RS_REMOTE_ACCESS_KEY=${MINIO_ACCESS_KEY} \ + --env RS_REMOTE_SECRET_KEY=${MINIO_SECRET_KEY} \ + --env RS_REMOTE_BUCKET=${MINIO_BUCKET} \ + --name reductstore -d ${{github.repository}} + - name: Upload data run: | pip install -r ./integration_tests/data_check/requirements.txt @@ -401,7 +474,7 @@ jobs: sleep 5 docker logs reductstore-1 - - name: Check data after migraiton + - name: Check data after migration run: python3 ./integration_tests/data_check/checker.py - name: Save docker logs @@ -410,7 +483,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: docker-log-recovery-${{matrix.cmd}} + name: docker-log-recovery-${{matrix.cmd}}-${{matrix.backend}} path: /tmp/docker-log.zip - name: Show replication report @@ -523,7 +596,7 @@ jobs: - unit_tests strategy: matrix: - branch: ["main", "latest"] + branch: [ "main", "latest" ] timeout-minutes: 5 steps: @@ -543,7 +616,6 @@ jobs: --name reductstore \ --env RS_DATA_PATH=/data \ --env RS_API_TOKEN=token \ - --env RS_LOG_LEVEL=DEBUG \ -p 8383:8383 \ -v ./data:/data \ ${{ github.repository }}:latest @@ -645,7 +717,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: [amd64, arm64] + arch: [ amd64, arm64 ] needs: - build_snap - api_tests @@ -678,7 +750,7 @@ jobs: if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') strategy: matrix: - platform: [linux/amd64, linux/arm64] + platform: [ linux/amd64, linux/arm64 ] include: - platform: linux/amd64 cargo_target: x86_64-unknown-linux-gnu @@ -727,6 +799,9 @@ jobs: GIT_COMMIT=${{env.GITHUB_SHA}} CARGO_TARGET=${{matrix.cargo_target}} GCC_COMPILER=${{matrix.gcc_compiler}} + ARTIFACT_SAS_URL=${{ secrets.ARTIFACT_SAS_URL }} + RUST_VERSION=${{ vars.MINIMAL_RUST_VERSION }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Export digest @@ -795,7 +870,7 @@ jobs: - name: Set up Rust uses: dtolnay/rust-toolchain@stable with: - toolchain: ${{ env.MINIMAL_RUST_VERSION }} + toolchain: ${{ vars.MINIMAL_RUST_VERSION }} - uses: arduino/setup-protoc@v1 with: version: "3.x" diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e77bfa4..257acd4fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,106 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Integrate S3 Storage Backend, [PR-919](https://github.com/reductstore/reductstore/pull/919) + +### Changed + +- Update extensions for Rust 1.89: ros-ext v0.3.0, select v0.5.0, [PR-921](https://github.com/reductstore/reductstore/pull/921) + +### Fixed + +- Add CA certificates to Docker image for AWS S3 access, [PR-924](https://github.com/reductstore/reductstore/pull/924) +- Fix path for token repository when cache used, [PR-925](https://github.com/reductstore/reductstore/pull/925) + +## [1.16.3] - 2025-08-22 + +### Fixed + +- Fix double synchronization of transaction log + integrity checks, [PR-912](https://github.com/reductstore/reductstore/pull/912) + +## [1.16.2] - 2025-08-22 + +### Fixed + +- Prevent data lost of unsynchronized files in case of power failure for unfinished blocks, [PR-909](https://github.com/reductstore/reductstore/pull/909) + +### Security + +- CVE-2025-55159: Update slab up to 0.4.11, [PR-911](https://github.com/reductstore/reductstore/pull/911) + +## [1.16.1] - 2025-08-09 + +### Fixed + +- Fix directive parsing and order of operators in conditional query, [PR-899](https://github.com/reductstore/reductstore/pull/899) + +### Changed + +- Update Web Console up to v1.11.2, [PR-900](https://github.com/reductstore/reductstore/pull/900) + +## [1.16.0] - 2025-07-31 + +### Added + +- Pass server information to extensions, [PR-816](https://github.com/reductstore/reductstore/pull/816) +- Integrate ReductSelect v0.1.0, [PR-821](https://github.com/reductstore/reductstore/pull/821) +- Integrate ReductRos v0.1.0, [PR-856](https://github.com/reductstore/reductstore/pull/856) +- Filter buckets by read permission in server information, [PR-849](https://github.com/reductstore/reductstore/pull/849) +- Support for duration literals, [PR-864](https://github.com/reductstore/reductstore/pull/864) +- Support for `#ctx_before` and `#ctx_after` directives, [PR-866](https://github.com/reductstore/reductstore/pull/866) +- Support for wildcards in write/read token permissions, [PR-877](https://github.com/reductstore/reductstore/pull/877) +- Check format of read/write token permissions, [PR-881](https://github.com/reductstore/reductstore/pull/881) +- Add support for selecting specific labels via the `#select_labels` directive, [PR-888](https://github.com/reductstore/reductstore/pull/888) + +### Changed + +- Add "@" prefix to computed labels, [PR-815](https://github.com/reductstore/reductstore/pull/815) +- Refactor Extension API for multi-line CSV processing, ReductSelect v0.2.0, [PR-823](https://github.com/reductstore/reductstore/pull/823) +- Run all operands after a compute-staged one on the compute stage, [PR-835](https://github.com/reductstore/reductstore/pull/835) +- Replace auto-staging for extension filtering with when condition in ext parameter, [PR-838](https://github.com/reductstore/reductstore/pull/838) +- Update ReductSelect up to v0.3.0, with CSV headers and data buffering, [PR-850](https://github.com/reductstore/reductstore/pull/850) +- Update ReductROS up to v0.2.0 with binary data encoding, [PR-882](https://github.com/reductstore/reductstore/pull/882) +- Update WebConsole up to v1.11.0 with many improvements, [PR-883](https://github.com/reductstore/reductstore/pull/883) +- Update ReductSelect up to v0.4.0, [PR-889](https://github.com/reductstore/reductstore/pull/889) +- Update Web Console up to v1.11.1 and ReductSelect up to 0.4.1, [PR-890](https://github.com/reductstore/reductstore/pull/890) + +### Fixed + +- Fix hanging query request if no extension registered, [PR-830](https://github.com/reductstore/reductstore/pull/830) +- Fix `$limit` operator in extension context, [PR-848](https://github.com/reductstore/reductstore/pull/848) +- Fix query finalization in ExtRepository, [PR-891](https://github.com/reductstore/reductstore/pull/891) + +### Removed + +- `x-reduct-last` header from query response, [PR-876](https://github.com/reductstore/reductstore/pull/876) + +## [1.15.6] - 2025-06-25 + +### Fixed + +- Fix replication of update transaction to empty bucket, [PR-867](https://github.com/reductstore/reductstore/pull/867) + +## [1.15.5] - 2025-06-10 + +### Fixed + +- Fix crash if RS_API_PATH has wrong format, [PR-846](https://github.com/reductstore/reductstore/pull/846) + +### Changed + +- Update Web Console up to 1.10.2, [PR-847](https://github.com/reductstore/reductstore/pull/847) + +## [1.15.4] - 2025-05-28 + +### Fixed + +- Update yanked zip dependency to 4.0.0, [PR-837](https://github.com/reductstore/reductstore/pull/837) + ## [1.15.3] - 2025-05-26 -## Fixed +### Fixed - Fix lock of write channel for small chunks, [PR-834](https://github.com/reductstore/reductstore/pull/834) @@ -43,7 +140,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - RS-645: Implement `$timestamp` operator, [PR-798](https://github.com/reductstore/reductstore/pull/798) - RS-646: Enable logging in extensions, [PR-646](https://github.com/reductstore/reductstore/pull/794) - ### Changed - Minimum Rust version to 1.85 @@ -186,7 +282,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update Web Console up to v1.8.0, [PR-655](https://github.com/reductstore/reductstore/pull/655) - ### Fixed - RS-544: Fix keeping quota error, [PR-654](https://github.com/reductstore/reductstore/pull/654) @@ -199,7 +294,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - RS-536: Update README.md, [PR-649](https://github.com/reductstore/reductstore/pull/649) - RS-193: Cross-compilation in CI/CD, [PR-651](https://github.com/reductstore/reductstore/pull/651) - ## [1.12.4] - 2024-11-20 ### Fixed @@ -688,7 +782,7 @@ reduct-rs: `ReductClient.url`, `ReductClient.token`, `ReductCientBuilder.try_bui ### Added -- Labels for `POST|GET /api/v1/:bucket/:entry` as headers with +- Labels for `POST|GET /api/v1/:bucket/:entry` as headers with prefix `x-reduct-label-`, [PR-224](https://github.com/reductstore/reductstore/pull/224) - `include-